2024-09-17T09:21:17.291 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-17T09:21:17.308 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-17T09:21:17.406 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/avan-2024-09-17_09:09:13-orch:cephadm-wip-athakkar-testing-2024-09-17-0048-distro-default-smithi/7908750 branch: wip-athakkar-testing-2024-09-17-0048 description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: athakkar@redhat.com first_in_suite: false job_id: '7908750' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: avan-2024-09-17_09:09:13-orch:cephadm-wip-athakkar-testing-2024-09-17-0048-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-athakkar-testing-2024-09-17-0048 ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - CEPHADM_REFRESH_FAILED log-only-match: - CEPHADM_ sha1: e646f72961223a04d41eb40dad875d1628d78516 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: e646f72961223a04d41eb40dad875d1628d78516 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: wip-athakkar-testing-2024-09-17-0048 sha1: e646f72961223a04d41eb40dad875d1628d78516 owner: scheduled_avan@teuthology priority: 80 repo: https://git.ceph.com/ceph-ci.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 5717 sha1: e646f72961223a04d41eb40dad875d1628d78516 sleep_before_teardown: 0 subset: 1/20 suite: orch:cephadm suite_branch: wip-athakkar-testing-2024-09-17-0048 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_e646f72961223a04d41eb40dad875d1628d78516/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: e646f72961223a04d41eb40dad875d1628d78516 targets: smithi078.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAl/vI0GIRfmqET2eTgsyVz/bGI0IlVii5+2JE/EdXmJUrUyTEnR5ms7rtqbayB58Tm/IKowq0LTtZwXo+jPIYA= smithi136.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDnU19b+ZRCqexdOmRNoB9pLL5vqrAlHKo/5SYMAUR25KGWRGlSyRAqbN2pex7DF3TVhIjq+4cKYTtj4SQYlCPY= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-17_09:09:13 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: avan verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-09-17T09:21:17.406 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_e646f72961223a04d41eb40dad875d1628d78516/qa; will attempt to use it 2024-09-17T09:21:17.407 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_e646f72961223a04d41eb40dad875d1628d78516/qa/tasks 2024-09-17T09:21:17.407 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-17T09:21:17.409 INFO:teuthology.task.internal:Checking packages... 2024-09-17T09:21:17.430 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'e646f72961223a04d41eb40dad875d1628d78516' 2024-09-17T09:21:17.430 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-17T09:21:17.430 INFO:teuthology.packaging:ref: None 2024-09-17T09:21:17.431 INFO:teuthology.packaging:tag: None 2024-09-17T09:21:17.431 INFO:teuthology.packaging:branch: wip-athakkar-testing-2024-09-17-0048 2024-09-17T09:21:17.431 INFO:teuthology.packaging:sha1: e646f72961223a04d41eb40dad875d1628d78516 2024-09-17T09:21:17.431 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-athakkar-testing-2024-09-17-0048 2024-09-17T09:21:17.573 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-4963.ge646f729 2024-09-17T09:21:17.575 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-17T09:21:17.580 INFO:teuthology.task.internal:no buildpackages task found 2024-09-17T09:21:17.580 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-17T09:21:17.586 INFO:teuthology.task.internal:Saving configuration 2024-09-17T09:21:17.600 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-17T09:21:17.606 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-17T09:21:17.630 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi078.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/avan-2024-09-17_09:09:13-orch:cephadm-wip-athakkar-testing-2024-09-17-0048-distro-default-smithi/7908750', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-17 09:16:17.841905', 'locked_by': 'scheduled_avan@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAl/vI0GIRfmqET2eTgsyVz/bGI0IlVii5+2JE/EdXmJUrUyTEnR5ms7rtqbayB58Tm/IKowq0LTtZwXo+jPIYA='} 2024-09-17T09:21:17.651 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi136.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/avan-2024-09-17_09:09:13-orch:cephadm-wip-athakkar-testing-2024-09-17-0048-distro-default-smithi/7908750', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-17 09:16:17.840558', 'locked_by': 'scheduled_avan@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDnU19b+ZRCqexdOmRNoB9pLL5vqrAlHKo/5SYMAUR25KGWRGlSyRAqbN2pex7DF3TVhIjq+4cKYTtj4SQYlCPY='} 2024-09-17T09:21:17.652 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-17T09:21:17.657 INFO:teuthology.task.internal:roles: ubuntu@smithi078.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-09-17T09:21:17.657 INFO:teuthology.task.internal:roles: ubuntu@smithi136.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-09-17T09:21:17.658 INFO:teuthology.run_tasks:Running task console_log... 2024-09-17T09:21:17.718 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f2e3172fac0>, signals=[15]) 2024-09-17T09:21:17.718 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-17T09:21:17.723 INFO:teuthology.task.internal:Opening connections... 2024-09-17T09:21:17.724 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi078.front.sepia.ceph.com 2024-09-17T09:21:17.725 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T09:21:17.802 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi136.front.sepia.ceph.com 2024-09-17T09:21:17.803 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi136.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T09:21:17.886 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-17T09:21:17.893 DEBUG:teuthology.orchestra.run.smithi078:> uname -m 2024-09-17T09:21:17.911 INFO:teuthology.orchestra.run.smithi078.stdout:x86_64 2024-09-17T09:21:17.911 DEBUG:teuthology.orchestra.run.smithi078:> cat /etc/os-release 2024-09-17T09:21:17.968 INFO:teuthology.orchestra.run.smithi078.stdout:NAME="CentOS Stream" 2024-09-17T09:21:17.968 INFO:teuthology.orchestra.run.smithi078.stdout:VERSION="9" 2024-09-17T09:21:17.968 INFO:teuthology.orchestra.run.smithi078.stdout:ID="centos" 2024-09-17T09:21:17.968 INFO:teuthology.orchestra.run.smithi078.stdout:ID_LIKE="rhel fedora" 2024-09-17T09:21:17.968 INFO:teuthology.orchestra.run.smithi078.stdout:VERSION_ID="9" 2024-09-17T09:21:17.969 INFO:teuthology.orchestra.run.smithi078.stdout:PLATFORM_ID="platform:el9" 2024-09-17T09:21:17.969 INFO:teuthology.orchestra.run.smithi078.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-17T09:21:17.969 INFO:teuthology.orchestra.run.smithi078.stdout:ANSI_COLOR="0;31" 2024-09-17T09:21:17.969 INFO:teuthology.orchestra.run.smithi078.stdout:LOGO="fedora-logo-icon" 2024-09-17T09:21:17.969 INFO:teuthology.orchestra.run.smithi078.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-17T09:21:17.969 INFO:teuthology.orchestra.run.smithi078.stdout:HOME_URL="https://centos.org/" 2024-09-17T09:21:17.969 INFO:teuthology.orchestra.run.smithi078.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-17T09:21:17.969 INFO:teuthology.orchestra.run.smithi078.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-17T09:21:17.969 INFO:teuthology.orchestra.run.smithi078.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-17T09:21:17.970 INFO:teuthology.lock.ops:Updating smithi078.front.sepia.ceph.com on lock server 2024-09-17T09:21:17.990 DEBUG:teuthology.orchestra.run.smithi136:> uname -m 2024-09-17T09:21:18.008 INFO:teuthology.orchestra.run.smithi136.stdout:x86_64 2024-09-17T09:21:18.008 DEBUG:teuthology.orchestra.run.smithi136:> cat /etc/os-release 2024-09-17T09:21:18.065 INFO:teuthology.orchestra.run.smithi136.stdout:NAME="CentOS Stream" 2024-09-17T09:21:18.065 INFO:teuthology.orchestra.run.smithi136.stdout:VERSION="9" 2024-09-17T09:21:18.065 INFO:teuthology.orchestra.run.smithi136.stdout:ID="centos" 2024-09-17T09:21:18.065 INFO:teuthology.orchestra.run.smithi136.stdout:ID_LIKE="rhel fedora" 2024-09-17T09:21:18.065 INFO:teuthology.orchestra.run.smithi136.stdout:VERSION_ID="9" 2024-09-17T09:21:18.065 INFO:teuthology.orchestra.run.smithi136.stdout:PLATFORM_ID="platform:el9" 2024-09-17T09:21:18.065 INFO:teuthology.orchestra.run.smithi136.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-17T09:21:18.066 INFO:teuthology.orchestra.run.smithi136.stdout:ANSI_COLOR="0;31" 2024-09-17T09:21:18.066 INFO:teuthology.orchestra.run.smithi136.stdout:LOGO="fedora-logo-icon" 2024-09-17T09:21:18.066 INFO:teuthology.orchestra.run.smithi136.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-17T09:21:18.066 INFO:teuthology.orchestra.run.smithi136.stdout:HOME_URL="https://centos.org/" 2024-09-17T09:21:18.066 INFO:teuthology.orchestra.run.smithi136.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-17T09:21:18.066 INFO:teuthology.orchestra.run.smithi136.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-17T09:21:18.066 INFO:teuthology.orchestra.run.smithi136.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-17T09:21:18.066 INFO:teuthology.lock.ops:Updating smithi136.front.sepia.ceph.com on lock server 2024-09-17T09:21:18.090 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-17T09:21:18.098 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-17T09:21:18.102 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-17T09:21:18.103 DEBUG:teuthology.orchestra.run.smithi078:> test '!' -e /home/ubuntu/cephtest 2024-09-17T09:21:18.105 DEBUG:teuthology.orchestra.run.smithi136:> test '!' -e /home/ubuntu/cephtest 2024-09-17T09:21:18.121 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-17T09:21:18.128 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-17T09:21:18.128 DEBUG:teuthology.orchestra.run.smithi078:> test -z $(ls -A /var/lib/ceph) 2024-09-17T09:21:18.163 DEBUG:teuthology.orchestra.run.smithi136:> test -z $(ls -A /var/lib/ceph) 2024-09-17T09:21:18.447 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-17T09:21:18.505 INFO:teuthology.run_tasks:Running task kernel... 2024-09-17T09:21:18.519 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-17T09:21:18.519 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-17T09:21:18.520 DEBUG:teuthology.orchestra.run.smithi078:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-17T09:21:18.520 DEBUG:teuthology.orchestra.run.smithi136:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-17T09:21:18.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T09:21:18.538 DEBUG:teuthology.orchestra.run.smithi078:> uname -r 2024-09-17T09:21:18.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T09:21:18.539 DEBUG:teuthology.orchestra.run.smithi136:> uname -r 2024-09-17T09:21:18.595 INFO:teuthology.orchestra.run.smithi078.stdout:5.14.0-503.el9.x86_64 2024-09-17T09:21:18.596 INFO:teuthology.task.kernel:Running kernel on smithi078: 5.14.0-503.el9.x86_64 2024-09-17T09:21:18.596 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum install -y kernel 2024-09-17T09:21:18.597 INFO:teuthology.orchestra.run.smithi136.stdout:5.14.0-503.el9.x86_64 2024-09-17T09:21:18.598 INFO:teuthology.task.kernel:Running kernel on smithi136: 5.14.0-503.el9.x86_64 2024-09-17T09:21:18.598 DEBUG:teuthology.orchestra.run.smithi136:> sudo yum install -y kernel 2024-09-17T09:21:21.861 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - BaseOS 5.1 MB/s | 8.3 MB 00:01 2024-09-17T09:21:22.179 INFO:teuthology.orchestra.run.smithi136.stdout:CentOS Stream 9 - BaseOS 4.7 MB/s | 8.3 MB 00:01 2024-09-17T09:21:23.554 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - AppStream 34 MB/s | 20 MB 00:00 2024-09-17T09:21:24.713 INFO:teuthology.orchestra.run.smithi136.stdout:CentOS Stream 9 - AppStream 14 MB/s | 20 MB 00:01 2024-09-17T09:21:28.738 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - CRB 4.6 MB/s | 6.5 MB 00:01 2024-09-17T09:21:30.339 INFO:teuthology.orchestra.run.smithi136.stdout:CentOS Stream 9 - CRB 3.8 MB/s | 6.5 MB 00:01 2024-09-17T09:21:30.413 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - Extras packages 40 kB/s | 19 kB 00:00 2024-09-17T09:21:31.485 INFO:teuthology.orchestra.run.smithi078.stdout:Extra Packages for Enterprise Linux 24 MB/s | 23 MB 00:00 2024-09-17T09:21:31.881 INFO:teuthology.orchestra.run.smithi136.stdout:CentOS Stream 9 - Extras packages 66 kB/s | 19 kB 00:00 2024-09-17T09:21:32.728 INFO:teuthology.orchestra.run.smithi136.stdout:Extra Packages for Enterprise Linux 31 MB/s | 23 MB 00:00 2024-09-17T09:21:36.899 INFO:teuthology.orchestra.run.smithi078.stdout:lab-extras 27 kB/s | 1.7 kB 00:00 2024-09-17T09:21:38.184 INFO:teuthology.orchestra.run.smithi136.stdout:lab-extras 31 kB/s | 1.7 kB 00:00 2024-09-17T09:21:38.789 INFO:teuthology.orchestra.run.smithi078.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-17T09:21:38.789 INFO:teuthology.orchestra.run.smithi078.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-17T09:21:38.866 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-17T09:21:38.877 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-17T09:21:38.877 INFO:teuthology.orchestra.run.smithi078.stdout: Package Architecture Version Repository Size 2024-09-17T09:21:38.877 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-17T09:21:38.877 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-09-17T09:21:38.877 INFO:teuthology.orchestra.run.smithi078.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-17T09:21:38.877 INFO:teuthology.orchestra.run.smithi078.stdout:Installing dependencies: 2024-09-17T09:21:38.877 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-17T09:21:38.877 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-17T09:21:38.878 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-17T09:21:38.878 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-17T09:21:38.878 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-17T09:21:38.878 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-17T09:21:38.878 INFO:teuthology.orchestra.run.smithi078.stdout:Install 4 Packages 2024-09-17T09:21:38.878 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-17T09:21:38.878 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 79 M 2024-09-17T09:21:38.879 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 126 M 2024-09-17T09:21:38.879 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-17T09:21:39.183 INFO:teuthology.orchestra.run.smithi078.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 127 kB/s | 36 kB 00:00 2024-09-17T09:21:40.107 INFO:teuthology.orchestra.run.smithi136.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-17T09:21:40.108 INFO:teuthology.orchestra.run.smithi136.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-17T09:21:40.207 INFO:teuthology.orchestra.run.smithi136.stdout:Dependencies resolved. 2024-09-17T09:21:40.214 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2024-09-17T09:21:40.215 INFO:teuthology.orchestra.run.smithi136.stdout: Package Architecture Version Repository Size 2024-09-17T09:21:40.215 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2024-09-17T09:21:40.215 INFO:teuthology.orchestra.run.smithi136.stdout:Installing: 2024-09-17T09:21:40.215 INFO:teuthology.orchestra.run.smithi136.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-17T09:21:40.215 INFO:teuthology.orchestra.run.smithi136.stdout:Installing dependencies: 2024-09-17T09:21:40.215 INFO:teuthology.orchestra.run.smithi136.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-17T09:21:40.215 INFO:teuthology.orchestra.run.smithi136.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-17T09:21:40.215 INFO:teuthology.orchestra.run.smithi136.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-17T09:21:40.215 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-17T09:21:40.215 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction Summary 2024-09-17T09:21:40.215 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2024-09-17T09:21:40.216 INFO:teuthology.orchestra.run.smithi136.stdout:Install 4 Packages 2024-09-17T09:21:40.216 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-17T09:21:40.216 INFO:teuthology.orchestra.run.smithi136.stdout:Total download size: 79 M 2024-09-17T09:21:40.216 INFO:teuthology.orchestra.run.smithi136.stdout:Installed size: 126 M 2024-09-17T09:21:40.216 INFO:teuthology.orchestra.run.smithi136.stdout:Downloading Packages: 2024-09-17T09:21:40.502 INFO:teuthology.orchestra.run.smithi136.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 272 kB/s | 36 kB 00:00 2024-09-17T09:21:41.528 INFO:teuthology.orchestra.run.smithi136.stdout:(2/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 30 MB/s | 35 MB 00:01 2024-09-17T09:21:41.678 INFO:teuthology.orchestra.run.smithi136.stdout:(3/4): kernel-core-5.14.0-508.el9.x86_64.rpm 12 MB/s | 16 MB 00:01 2024-09-17T09:21:42.170 INFO:teuthology.orchestra.run.smithi136.stdout:(4/4): kernel-modules-core-5.14.0-508.el9.x86_6 17 MB/s | 29 MB 00:01 2024-09-17T09:21:42.170 INFO:teuthology.orchestra.run.smithi136.stdout:-------------------------------------------------------------------------------- 2024-09-17T09:21:42.170 INFO:teuthology.orchestra.run.smithi136.stdout:Total 40 MB/s | 79 MB 00:01 2024-09-17T09:21:42.857 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction check 2024-09-17T09:21:43.206 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction check succeeded. 2024-09-17T09:21:43.207 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction test 2024-09-17T09:21:44.172 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction test succeeded. 2024-09-17T09:21:44.172 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction 2024-09-17T09:21:46.591 INFO:teuthology.orchestra.run.smithi136.stdout: Preparing : 1/1 2024-09-17T09:21:47.303 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-17T09:21:47.386 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T09:21:50.146 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T09:21:50.277 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T09:21:54.107 INFO:teuthology.orchestra.run.smithi078.stdout:(2/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 2.3 MB/s | 35 MB 00:15 2024-09-17T09:21:55.100 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T09:21:55.152 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-17T09:21:55.183 INFO:teuthology.orchestra.run.smithi078.stdout:(3/4): kernel-core-5.14.0-508.el9.x86_64.rpm 985 kB/s | 16 MB 00:16 2024-09-17T09:21:55.650 INFO:teuthology.orchestra.run.smithi078.stdout:(4/4): kernel-modules-core-5.14.0-508.el9.x86_6 1.7 MB/s | 29 MB 00:16 2024-09-17T09:21:55.650 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-17T09:21:55.650 INFO:teuthology.orchestra.run.smithi078.stdout:Total 4.7 MB/s | 79 MB 00:16 2024-09-17T09:21:56.331 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-17T09:21:56.703 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-17T09:21:56.703 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-17T09:21:57.682 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-17T09:21:57.682 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-17T09:22:00.064 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T09:22:00.153 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-17T09:22:00.803 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-17T09:22:00.893 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T09:22:03.654 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T09:22:03.789 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T09:22:08.613 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T09:22:08.655 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-17T09:22:13.508 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T09:22:38.393 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T09:22:38.438 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-17T09:22:38.768 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-17T09:22:38.768 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-17T09:22:38.768 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T09:22:38.768 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T09:22:39.606 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T09:22:39.606 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-17T09:22:39.606 INFO:teuthology.orchestra.run.smithi136.stdout:Installed: 2024-09-17T09:22:39.606 INFO:teuthology.orchestra.run.smithi136.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-17T09:22:39.607 INFO:teuthology.orchestra.run.smithi136.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-17T09:22:39.607 INFO:teuthology.orchestra.run.smithi136.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-17T09:22:39.607 INFO:teuthology.orchestra.run.smithi136.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-17T09:22:39.607 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-17T09:22:39.607 INFO:teuthology.orchestra.run.smithi136.stdout:Complete! 2024-09-17T09:22:39.961 DEBUG:teuthology.orchestra.run.smithi136:> rpm -q kernel | sort -rV | head -n 1 2024-09-17T09:22:40.042 INFO:teuthology.orchestra.run.smithi136.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-17T09:22:40.042 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-17T09:22:40.043 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-17T09:22:40.043 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-17T09:22:40.043 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-17T09:22:40.043 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-09-17T09:22:40.043 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi136.front.sepia.ceph.com, path=None, version=distro) 2024-09-17T09:22:40.043 DEBUG:teuthology.orchestra.run.smithi136:> rpm -q kernel | sort -rV | head -n 1 2024-09-17T09:22:40.101 INFO:teuthology.orchestra.run.smithi136.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-17T09:22:40.102 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-17T09:22:40.102 DEBUG:teuthology.orchestra.run.smithi136:> sudo rpm -qi grub2-tools 2024-09-17T09:22:40.134 INFO:teuthology.orchestra.run.smithi136.stdout:Name : grub2-tools 2024-09-17T09:22:40.134 INFO:teuthology.orchestra.run.smithi136.stdout:Epoch : 1 2024-09-17T09:22:40.134 INFO:teuthology.orchestra.run.smithi136.stdout:Version : 2.06 2024-09-17T09:22:40.134 INFO:teuthology.orchestra.run.smithi136.stdout:Release : 82.el9 2024-09-17T09:22:40.134 INFO:teuthology.orchestra.run.smithi136.stdout:Architecture: x86_64 2024-09-17T09:22:40.134 INFO:teuthology.orchestra.run.smithi136.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-17T09:22:40.134 INFO:teuthology.orchestra.run.smithi136.stdout:Group : Unspecified 2024-09-17T09:22:40.135 INFO:teuthology.orchestra.run.smithi136.stdout:Size : 8274670 2024-09-17T09:22:40.135 INFO:teuthology.orchestra.run.smithi136.stdout:License : GPLv3+ 2024-09-17T09:22:40.135 INFO:teuthology.orchestra.run.smithi136.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-17T09:22:40.135 INFO:teuthology.orchestra.run.smithi136.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-17T09:22:40.135 INFO:teuthology.orchestra.run.smithi136.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-17T09:22:40.135 INFO:teuthology.orchestra.run.smithi136.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-17T09:22:40.135 INFO:teuthology.orchestra.run.smithi136.stdout:Packager : builder@centos.org 2024-09-17T09:22:40.135 INFO:teuthology.orchestra.run.smithi136.stdout:Vendor : CentOS 2024-09-17T09:22:40.135 INFO:teuthology.orchestra.run.smithi136.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-17T09:22:40.135 INFO:teuthology.orchestra.run.smithi136.stdout:Summary : Support tools for GRUB. 2024-09-17T09:22:40.135 INFO:teuthology.orchestra.run.smithi136.stdout:Description : 2024-09-17T09:22:40.135 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-17T09:22:40.135 INFO:teuthology.orchestra.run.smithi136.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-17T09:22:40.136 INFO:teuthology.orchestra.run.smithi136.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-17T09:22:40.136 INFO:teuthology.orchestra.run.smithi136.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-17T09:22:40.136 INFO:teuthology.orchestra.run.smithi136.stdout:hardware devices. 2024-09-17T09:22:40.136 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-17T09:22:40.136 INFO:teuthology.orchestra.run.smithi136.stdout:This subpackage provides tools for support of all platforms. 2024-09-17T09:22:40.137 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-17T09:22:40.137 INFO:teuthology.task.kernel:Updating grub on smithi136 to boot 5.14.0-508.el9.x86_64 2024-09-17T09:22:40.137 DEBUG:teuthology.orchestra.run.smithi136:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-17T09:22:40.808 INFO:teuthology.orchestra.run.smithi136.stderr:Generating grub configuration file ... 2024-09-17T09:22:42.140 INFO:teuthology.orchestra.run.smithi136.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-17T09:22:42.175 INFO:teuthology.orchestra.run.smithi136.stderr:done 2024-09-17T09:22:42.178 DEBUG:teuthology.orchestra.run.smithi136:> mktemp 2024-09-17T09:22:42.193 INFO:teuthology.orchestra.run.smithi136.stdout:/tmp/tmp.zaa82Ga4N2 2024-09-17T09:22:42.194 DEBUG:teuthology.orchestra.run.smithi136:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.zaa82Ga4N2 2024-09-17T09:22:42.260 DEBUG:teuthology.orchestra.run.smithi136:> sudo chmod 0666 /tmp/tmp.zaa82Ga4N2 2024-09-17T09:22:42.403 DEBUG:teuthology.orchestra.remote:smithi136:/tmp/tmp.zaa82Ga4N2 is 6KB 2024-09-17T09:22:42.451 DEBUG:teuthology.orchestra.run.smithi136:> rm -fr /tmp/tmp.zaa82Ga4N2 2024-09-17T09:22:42.466 DEBUG:teuthology.orchestra.run.smithi136:> sudo /bin/ls /boot/loader/entries || true 2024-09-17T09:22:42.531 INFO:teuthology.orchestra.run.smithi136.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-17T09:22:42.531 INFO:teuthology.orchestra.run.smithi136.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-17T09:22:42.532 INFO:teuthology.orchestra.run.smithi136.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-17T09:22:42.532 INFO:teuthology.orchestra.run.smithi136.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-17T09:22:42.533 DEBUG:teuthology.orchestra.run.smithi136:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-17T09:22:42.681 DEBUG:teuthology.orchestra.run.smithi136:> sudo shutdown -r now 2024-09-17T09:22:51.542 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T09:22:51.582 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-17T09:22:51.965 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-17T09:22:51.966 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-17T09:22:51.966 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T09:22:51.966 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T09:22:52.817 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T09:22:52.817 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-17T09:22:52.817 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-17T09:22:52.817 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-17T09:22:52.817 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-17T09:22:52.817 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-17T09:22:52.817 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-17T09:22:52.817 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-17T09:22:52.817 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-17T09:22:53.164 DEBUG:teuthology.orchestra.run.smithi078:> rpm -q kernel | sort -rV | head -n 1 2024-09-17T09:22:53.247 INFO:teuthology.orchestra.run.smithi078.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-17T09:22:53.247 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-17T09:22:53.247 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-17T09:22:53.247 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-17T09:22:53.247 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-17T09:22:53.247 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-09-17T09:22:53.247 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi078.front.sepia.ceph.com, path=None, version=distro) 2024-09-17T09:22:53.247 DEBUG:teuthology.orchestra.run.smithi078:> rpm -q kernel | sort -rV | head -n 1 2024-09-17T09:22:53.307 INFO:teuthology.orchestra.run.smithi078.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-17T09:22:53.307 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-17T09:22:53.308 DEBUG:teuthology.orchestra.run.smithi078:> sudo rpm -qi grub2-tools 2024-09-17T09:22:53.339 INFO:teuthology.orchestra.run.smithi078.stdout:Name : grub2-tools 2024-09-17T09:22:53.339 INFO:teuthology.orchestra.run.smithi078.stdout:Epoch : 1 2024-09-17T09:22:53.339 INFO:teuthology.orchestra.run.smithi078.stdout:Version : 2.06 2024-09-17T09:22:53.339 INFO:teuthology.orchestra.run.smithi078.stdout:Release : 82.el9 2024-09-17T09:22:53.339 INFO:teuthology.orchestra.run.smithi078.stdout:Architecture: x86_64 2024-09-17T09:22:53.340 INFO:teuthology.orchestra.run.smithi078.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-17T09:22:53.340 INFO:teuthology.orchestra.run.smithi078.stdout:Group : Unspecified 2024-09-17T09:22:53.340 INFO:teuthology.orchestra.run.smithi078.stdout:Size : 8274670 2024-09-17T09:22:53.340 INFO:teuthology.orchestra.run.smithi078.stdout:License : GPLv3+ 2024-09-17T09:22:53.340 INFO:teuthology.orchestra.run.smithi078.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-17T09:22:53.340 INFO:teuthology.orchestra.run.smithi078.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-17T09:22:53.340 INFO:teuthology.orchestra.run.smithi078.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-17T09:22:53.340 INFO:teuthology.orchestra.run.smithi078.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-17T09:22:53.340 INFO:teuthology.orchestra.run.smithi078.stdout:Packager : builder@centos.org 2024-09-17T09:22:53.340 INFO:teuthology.orchestra.run.smithi078.stdout:Vendor : CentOS 2024-09-17T09:22:53.340 INFO:teuthology.orchestra.run.smithi078.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-17T09:22:53.340 INFO:teuthology.orchestra.run.smithi078.stdout:Summary : Support tools for GRUB. 2024-09-17T09:22:53.340 INFO:teuthology.orchestra.run.smithi078.stdout:Description : 2024-09-17T09:22:53.341 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-17T09:22:53.341 INFO:teuthology.orchestra.run.smithi078.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-17T09:22:53.341 INFO:teuthology.orchestra.run.smithi078.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-17T09:22:53.341 INFO:teuthology.orchestra.run.smithi078.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-17T09:22:53.341 INFO:teuthology.orchestra.run.smithi078.stdout:hardware devices. 2024-09-17T09:22:53.341 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-17T09:22:53.341 INFO:teuthology.orchestra.run.smithi078.stdout:This subpackage provides tools for support of all platforms. 2024-09-17T09:22:53.343 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-17T09:22:53.343 INFO:teuthology.task.kernel:Updating grub on smithi078 to boot 5.14.0-508.el9.x86_64 2024-09-17T09:22:53.343 DEBUG:teuthology.orchestra.run.smithi078:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-17T09:22:54.046 INFO:teuthology.orchestra.run.smithi078.stderr:Generating grub configuration file ... 2024-09-17T09:22:55.364 INFO:teuthology.orchestra.run.smithi078.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-17T09:22:55.397 INFO:teuthology.orchestra.run.smithi078.stderr:done 2024-09-17T09:22:55.399 DEBUG:teuthology.orchestra.run.smithi078:> mktemp 2024-09-17T09:22:55.417 INFO:teuthology.orchestra.run.smithi078.stdout:/tmp/tmp.ZHZM3TLbdd 2024-09-17T09:22:55.417 DEBUG:teuthology.orchestra.run.smithi078:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.ZHZM3TLbdd 2024-09-17T09:22:55.485 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 0666 /tmp/tmp.ZHZM3TLbdd 2024-09-17T09:22:55.634 DEBUG:teuthology.orchestra.remote:smithi078:/tmp/tmp.ZHZM3TLbdd is 6KB 2024-09-17T09:22:55.683 DEBUG:teuthology.orchestra.run.smithi078:> rm -fr /tmp/tmp.ZHZM3TLbdd 2024-09-17T09:22:55.698 DEBUG:teuthology.orchestra.run.smithi078:> sudo /bin/ls /boot/loader/entries || true 2024-09-17T09:22:55.761 INFO:teuthology.orchestra.run.smithi078.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-17T09:22:55.761 INFO:teuthology.orchestra.run.smithi078.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-17T09:22:55.761 INFO:teuthology.orchestra.run.smithi078.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-17T09:22:55.761 INFO:teuthology.orchestra.run.smithi078.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-17T09:22:55.762 DEBUG:teuthology.orchestra.run.smithi078:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-17T09:22:55.930 DEBUG:teuthology.orchestra.run.smithi078:> sudo shutdown -r now 2024-09-17T09:23:12.698 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-09-17T09:23:12.699 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi136.front.sepia.ceph.com' 2024-09-17T09:23:12.700 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi136.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T09:23:25.935 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-09-17T09:23:25.935 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-17T09:23:25.936 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T09:23:31.261 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.136 2024-09-17T09:23:40.270 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi136.front.sepia.ceph.com' 2024-09-17T09:23:40.271 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi136.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T09:23:43.325 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.136 2024-09-17T09:23:44.317 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.78 2024-09-17T09:23:53.325 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-17T09:23:53.326 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T09:23:55.348 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi136.front.sepia.ceph.com' 2024-09-17T09:23:55.349 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi136.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T09:23:56.381 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.78 2024-09-17T09:24:08.393 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-17T09:24:08.394 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T09:24:13.757 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.136 2024-09-17T09:24:28.770 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi136.front.sepia.ceph.com' 2024-09-17T09:24:28.771 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi136.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T09:24:29.197 DEBUG:teuthology.orchestra.run.smithi136:> true 2024-09-17T09:24:29.574 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi136.front.sepia.ceph.com' 2024-09-17T09:24:29.574 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-508.el9.x86_64"... 2024-09-17T09:24:29.575 DEBUG:teuthology.orchestra.run.smithi136:> uname -r 2024-09-17T09:24:29.589 INFO:teuthology.orchestra.run.smithi136.stdout:5.14.0-508.el9.x86_64 2024-09-17T09:24:29.590 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-17T09:24:29.590 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-17T09:24:29.590 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-17T09:24:30.590 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-09-17T09:24:30.591 DEBUG:teuthology.orchestra.run.smithi136:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-17T09:24:30.849 INFO:teuthology.orchestra.run.smithi136.stdout:ttyS1 2024-09-17T09:24:30.874 DEBUG:teuthology.parallel:result is None 2024-09-17T09:24:40.339 DEBUG:teuthology.orchestra.run.smithi078:> true 2024-09-17T09:24:40.912 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-17T09:24:40.912 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-508.el9.x86_64"... 2024-09-17T09:24:40.912 DEBUG:teuthology.orchestra.run.smithi078:> uname -r 2024-09-17T09:24:40.954 INFO:teuthology.orchestra.run.smithi078.stdout:5.14.0-508.el9.x86_64 2024-09-17T09:24:40.954 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-17T09:24:40.954 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-17T09:24:40.954 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-17T09:24:41.955 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-09-17T09:24:41.955 DEBUG:teuthology.orchestra.run.smithi078:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-17T09:24:42.160 INFO:teuthology.orchestra.run.smithi078.stdout:ttyS1 2024-09-17T09:24:42.179 DEBUG:teuthology.parallel:result is None 2024-09-17T09:24:42.180 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-17T09:24:42.187 INFO:teuthology.task.internal:Creating test directory... 2024-09-17T09:24:42.187 DEBUG:teuthology.orchestra.run.smithi078:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-17T09:24:42.189 DEBUG:teuthology.orchestra.run.smithi136:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-17T09:24:42.208 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-17T09:24:42.214 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-17T09:24:42.223 INFO:teuthology.task.internal:Creating archive directory... 2024-09-17T09:24:42.223 DEBUG:teuthology.orchestra.run.smithi078:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-17T09:24:42.245 DEBUG:teuthology.orchestra.run.smithi136:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-17T09:24:42.352 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-17T09:24:42.359 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-17T09:24:42.359 DEBUG:teuthology.orchestra.run.smithi078:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-17T09:24:42.362 DEBUG:teuthology.orchestra.run.smithi136:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-17T09:24:42.390 INFO:teuthology.orchestra.run.smithi078.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-17T09:24:42.392 INFO:teuthology.orchestra.run.smithi136.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-17T09:24:42.400 INFO:teuthology.orchestra.run.smithi078.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-17T09:24:42.403 INFO:teuthology.orchestra.run.smithi136.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-17T09:24:42.405 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-17T09:24:42.412 INFO:teuthology.task.internal:Configuring sudo... 2024-09-17T09:24:42.412 DEBUG:teuthology.orchestra.run.smithi078:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-17T09:24:42.444 DEBUG:teuthology.orchestra.run.smithi136:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-17T09:24:42.472 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-17T09:24:42.483 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-17T09:24:42.484 DEBUG:teuthology.orchestra.run.smithi078:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-17T09:24:42.508 DEBUG:teuthology.orchestra.run.smithi136:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-17T09:24:42.530 DEBUG:teuthology.orchestra.run.smithi078:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-17T09:24:42.606 DEBUG:teuthology.orchestra.run.smithi078:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-17T09:24:42.657 DEBUG:teuthology.orchestra.run.smithi078:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-17T09:24:42.739 DEBUG:teuthology.orchestra.run.smithi078:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-17T09:24:42.779 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-17T09:24:42.779 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-17T09:24:42.843 DEBUG:teuthology.orchestra.run.smithi136:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-17T09:24:42.888 DEBUG:teuthology.orchestra.run.smithi136:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-17T09:24:42.938 DEBUG:teuthology.orchestra.run.smithi136:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-17T09:24:43.021 DEBUG:teuthology.orchestra.run.smithi136:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-17T09:24:43.064 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2024-09-17T09:24:43.064 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-17T09:24:43.130 DEBUG:teuthology.orchestra.run.smithi078:> sudo service rsyslog restart 2024-09-17T09:24:43.133 DEBUG:teuthology.orchestra.run.smithi136:> sudo service rsyslog restart 2024-09-17T09:24:43.179 INFO:teuthology.orchestra.run.smithi078.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-17T09:24:43.230 INFO:teuthology.orchestra.run.smithi136.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-17T09:24:43.631 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-17T09:24:43.638 INFO:teuthology.task.internal:Starting timer... 2024-09-17T09:24:43.638 INFO:teuthology.run_tasks:Running task pcp... 2024-09-17T09:24:43.648 INFO:teuthology.run_tasks:Running task selinux... 2024-09-17T09:24:43.656 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-09-17T09:24:43.657 DEBUG:teuthology.orchestra.run.smithi078:> sudo service auditd rotate 2024-09-17T09:24:43.724 INFO:teuthology.orchestra.run.smithi078.stdout:Rotating logs: 2024-09-17T09:24:43.726 DEBUG:teuthology.orchestra.run.smithi136:> sudo service auditd rotate 2024-09-17T09:24:43.789 INFO:teuthology.orchestra.run.smithi136.stdout:Rotating logs: 2024-09-17T09:24:43.790 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-17T09:24:43.791 DEBUG:teuthology.orchestra.run.smithi078:> /usr/sbin/getenforce 2024-09-17T09:24:43.822 INFO:teuthology.orchestra.run.smithi078.stdout:Permissive 2024-09-17T09:24:43.823 DEBUG:teuthology.orchestra.run.smithi136:> /usr/sbin/getenforce 2024-09-17T09:24:43.857 INFO:teuthology.orchestra.run.smithi136.stdout:Permissive 2024-09-17T09:24:43.857 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi078.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi136.front.sepia.ceph.com': 'permissive'} 2024-09-17T09:24:43.857 DEBUG:teuthology.orchestra.run.smithi078:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-17T09:24:43.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T09:24:43.890 DEBUG:teuthology.orchestra.run.smithi136:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-17T09:24:43.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T09:24:43.925 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-17T09:24:43.925 DEBUG:teuthology.orchestra.run.smithi078:> sudo /usr/sbin/setenforce permissive 2024-09-17T09:24:43.971 DEBUG:teuthology.orchestra.run.smithi136:> sudo /usr/sbin/setenforce permissive 2024-09-17T09:24:44.007 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-17T09:24:44.016 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-09-17T09:24:44.020 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-09-17T09:24:44.138 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-17T09:24:44.149 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-17T09:24:44.150 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi078.front.sepia.ceph.com,smithi136.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-17T09:28:33.406 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi078.front.sepia.ceph.com'), Remote(name='ubuntu@smithi136.front.sepia.ceph.com')] 2024-09-17T09:28:33.407 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-17T09:28:33.409 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T09:28:33.488 DEBUG:teuthology.orchestra.run.smithi078:> true 2024-09-17T09:28:33.568 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-17T09:28:33.569 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi136.front.sepia.ceph.com' 2024-09-17T09:28:33.569 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi136.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T09:28:33.648 DEBUG:teuthology.orchestra.run.smithi136:> true 2024-09-17T09:28:33.724 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi136.front.sepia.ceph.com' 2024-09-17T09:28:33.724 INFO:teuthology.run_tasks:Running task clock... 2024-09-17T09:28:33.735 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-17T09:28:33.735 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-17T09:28:33.736 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-17T09:28:33.738 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-17T09:28:33.738 DEBUG:teuthology.orchestra.run.smithi136:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-17T09:28:33.766 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-17T09:28:33.781 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-17T09:28:33.793 INFO:teuthology.orchestra.run.smithi136.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-17T09:28:33.805 INFO:teuthology.orchestra.run.smithi078.stderr:sudo: ntpd: command not found 2024-09-17T09:28:33.807 INFO:teuthology.orchestra.run.smithi136.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-17T09:28:33.816 INFO:teuthology.orchestra.run.smithi078.stdout:506 Cannot talk to daemon 2024-09-17T09:28:33.831 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-17T09:28:33.831 INFO:teuthology.orchestra.run.smithi136.stderr:sudo: ntpd: command not found 2024-09-17T09:28:33.843 INFO:teuthology.orchestra.run.smithi136.stdout:506 Cannot talk to daemon 2024-09-17T09:28:33.845 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-17T09:28:33.857 INFO:teuthology.orchestra.run.smithi136.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-17T09:28:33.870 INFO:teuthology.orchestra.run.smithi136.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-17T09:28:33.901 INFO:teuthology.orchestra.run.smithi078.stderr:bash: line 1: ntpq: command not found 2024-09-17T09:28:33.905 INFO:teuthology.orchestra.run.smithi078.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-17T09:28:33.905 INFO:teuthology.orchestra.run.smithi078.stdout:=============================================================================== 2024-09-17T09:28:33.905 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T09:28:33.905 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T09:28:33.905 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T09:28:33.905 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T09:28:33.925 INFO:teuthology.orchestra.run.smithi136.stderr:bash: line 1: ntpq: command not found 2024-09-17T09:28:33.929 INFO:teuthology.orchestra.run.smithi136.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-17T09:28:33.929 INFO:teuthology.orchestra.run.smithi136.stdout:=============================================================================== 2024-09-17T09:28:33.930 INFO:teuthology.orchestra.run.smithi136.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T09:28:33.930 INFO:teuthology.orchestra.run.smithi136.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T09:28:33.930 INFO:teuthology.orchestra.run.smithi136.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T09:28:33.930 INFO:teuthology.orchestra.run.smithi136.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T09:28:33.930 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-17T09:28:34.038 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'CEPHADM_REFRESH_FAILED'], 'log-only-match': ['CEPHADM_'], 'sha1': 'e646f72961223a04d41eb40dad875d1628d78516'} 2024-09-17T09:28:34.038 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-09-17T09:28:34.039 INFO:tasks.cephadm:Cluster fsid is 35c0cd22-74d7-11ef-bceb-c7b262605968 2024-09-17T09:28:34.039 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-17T09:28:34.039 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-09-17T09:28:34.039 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi078': '172.21.15.78', 'mon.smithi136': '172.21.15.136'} 2024-09-17T09:28:34.040 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-17T09:28:34.040 DEBUG:teuthology.orchestra.run.smithi078:> sudo hostname $(hostname -s) 2024-09-17T09:28:34.066 DEBUG:teuthology.orchestra.run.smithi136:> sudo hostname $(hostname -s) 2024-09-17T09:28:34.091 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-09-17T09:28:34.092 DEBUG:teuthology.orchestra.run.smithi078:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-17T09:28:34.435 INFO:teuthology.orchestra.run.smithi078.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 17 09:28 /home/ubuntu/cephtest/cephadm 2024-09-17T09:28:34.435 DEBUG:teuthology.orchestra.run.smithi136:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-17T09:28:34.715 INFO:teuthology.orchestra.run.smithi136.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 17 09:28 /home/ubuntu/cephtest/cephadm 2024-09-17T09:28:34.715 DEBUG:teuthology.orchestra.run.smithi078:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-17T09:28:34.732 DEBUG:teuthology.orchestra.run.smithi136:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-17T09:28:34.757 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-09-17T09:28:34.757 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-17T09:28:34.775 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-17T09:28:36.424 INFO:teuthology.orchestra.run.smithi078.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-17T09:28:36.467 INFO:teuthology.orchestra.run.smithi136.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-17T09:28:48.279 INFO:teuthology.orchestra.run.smithi136.stdout:{ 2024-09-17T09:28:48.279 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-17T09:28:48.279 INFO:teuthology.orchestra.run.smithi136.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-17T09:28:48.279 INFO:teuthology.orchestra.run.smithi136.stdout: "repo_digests": [ 2024-09-17T09:28:48.279 INFO:teuthology.orchestra.run.smithi136.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-17T09:28:48.279 INFO:teuthology.orchestra.run.smithi136.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-17T09:28:48.279 INFO:teuthology.orchestra.run.smithi136.stdout: ] 2024-09-17T09:28:48.280 INFO:teuthology.orchestra.run.smithi136.stdout:} 2024-09-17T09:28:49.120 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:28:49.120 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-17T09:28:49.121 INFO:teuthology.orchestra.run.smithi078.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-17T09:28:49.121 INFO:teuthology.orchestra.run.smithi078.stdout: "repo_digests": [ 2024-09-17T09:28:49.121 INFO:teuthology.orchestra.run.smithi078.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-17T09:28:49.121 INFO:teuthology.orchestra.run.smithi078.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-17T09:28:49.121 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-09-17T09:28:49.121 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:28:49.138 DEBUG:teuthology.orchestra.run.smithi078:> sudo mkdir -p /etc/ceph 2024-09-17T09:28:49.178 DEBUG:teuthology.orchestra.run.smithi136:> sudo mkdir -p /etc/ceph 2024-09-17T09:28:49.212 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 777 /etc/ceph 2024-09-17T09:28:49.243 DEBUG:teuthology.orchestra.run.smithi136:> sudo chmod 777 /etc/ceph 2024-09-17T09:28:49.280 INFO:tasks.cephadm:Writing seed config... 2024-09-17T09:28:49.283 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-17T09:28:49.283 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-17T09:28:49.284 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-17T09:28:49.284 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-17T09:28:49.284 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-17T09:28:49.284 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-17T09:28:49.284 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-17T09:28:49.284 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-17T09:28:49.284 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-17T09:28:49.285 DEBUG:teuthology.orchestra.run.smithi078:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-17T09:28:49.301 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 35c0cd22-74d7-11ef-bceb-c7b262605968 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-17T09:28:49.302 DEBUG:teuthology.orchestra.run.smithi078:mon.smithi078> sudo journalctl -f -n 0 -u ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@mon.smithi078.service 2024-09-17T09:28:49.345 INFO:tasks.cephadm:Bootstrapping... 2024-09-17T09:28:49.345 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.78 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-17T09:28:49.532 INFO:teuthology.orchestra.run.smithi078.stderr:-------------------------------------------------------------------------------- 2024-09-17T09:28:49.533 INFO:teuthology.orchestra.run.smithi078.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '35c0cd22-74d7-11ef-bceb-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.78', '--skip-admin-label'] 2024-09-17T09:28:49.554 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: 5.2.2 2024-09-17T09:28:49.556 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying podman|docker is present... 2024-09-17T09:28:49.581 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: 5.2.2 2024-09-17T09:28:49.583 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying lvm2 is present... 2024-09-17T09:28:49.583 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying time synchronization is in place... 2024-09-17T09:28:49.590 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-17T09:28:49.597 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2024-09-17T09:28:49.604 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: enabled 2024-09-17T09:28:49.610 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: active 2024-09-17T09:28:49.611 INFO:teuthology.orchestra.run.smithi078.stderr:Unit chronyd.service is enabled and running 2024-09-17T09:28:49.611 INFO:teuthology.orchestra.run.smithi078.stderr:Repeating the final host check... 2024-09-17T09:28:49.630 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: 5.2.2 2024-09-17T09:28:49.632 INFO:teuthology.orchestra.run.smithi078.stderr:podman (/bin/podman) version 5.2.2 is present 2024-09-17T09:28:49.632 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl is present 2024-09-17T09:28:49.632 INFO:teuthology.orchestra.run.smithi078.stderr:lvcreate is present 2024-09-17T09:28:49.638 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-17T09:28:49.645 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2024-09-17T09:28:49.652 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: enabled 2024-09-17T09:28:49.658 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: active 2024-09-17T09:28:49.659 INFO:teuthology.orchestra.run.smithi078.stderr:Unit chronyd.service is enabled and running 2024-09-17T09:28:49.659 INFO:teuthology.orchestra.run.smithi078.stderr:Host looks OK 2024-09-17T09:28:49.659 INFO:teuthology.orchestra.run.smithi078.stderr:Cluster fsid: 35c0cd22-74d7-11ef-bceb-c7b262605968 2024-09-17T09:28:49.659 INFO:teuthology.orchestra.run.smithi078.stderr:Acquiring lock 139815058139024 on /run/cephadm/35c0cd22-74d7-11ef-bceb-c7b262605968.lock 2024-09-17T09:28:49.659 INFO:teuthology.orchestra.run.smithi078.stderr:Lock 139815058139024 acquired on /run/cephadm/35c0cd22-74d7-11ef-bceb-c7b262605968.lock 2024-09-17T09:28:49.660 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying IP 172.21.15.78 port 3300 ... 2024-09-17T09:28:49.660 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying IP 172.21.15.78 port 6789 ... 2024-09-17T09:28:49.660 INFO:teuthology.orchestra.run.smithi078.stderr:Base mon IP is 172.21.15.78, final addrv is [v2:172.21.15.78:3300,v1:172.21.15.78:6789] 2024-09-17T09:28:49.662 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.78 metric 100 2024-09-17T09:28:49.662 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.78 metric 100 2024-09-17T09:28:49.665 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-09-17T09:28:49.665 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2024-09-17T09:28:49.665 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2024-09-17T09:28:49.668 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-17T09:28:49.668 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-09-17T09:28:49.668 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-17T09:28:49.668 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-09-17T09:28:49.668 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe8f:d4dd/64 scope link noprefixroute 2024-09-17T09:28:49.668 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-17T09:28:49.669 INFO:teuthology.orchestra.run.smithi078.stderr:Mon IP `172.21.15.78` is in CIDR network `172.21.0.0/20` 2024-09-17T09:28:49.669 INFO:teuthology.orchestra.run.smithi078.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-17T09:28:49.669 INFO:teuthology.orchestra.run.smithi078.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-17T09:28:49.769 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-09-17T09:28:50.111 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Getting image source signatures 2024-09-17T09:28:50.111 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-09-17T09:28:50.111 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-09-17T09:28:50.111 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-09-17T09:28:50.111 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-09-17T09:28:50.111 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-09-17T09:28:50.111 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-17T09:28:50.248 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Writing manifest to image destination 2024-09-17T09:28:50.368 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-17T09:28:50.784 INFO:teuthology.orchestra.run.smithi078.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-17T09:28:51.292 INFO:teuthology.orchestra.run.smithi078.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-17T09:28:51.292 INFO:teuthology.orchestra.run.smithi078.stderr:Extracting ceph user uid/gid from container image... 2024-09-17T09:28:51.682 INFO:teuthology.orchestra.run.smithi078.stderr:stat: 167 167 2024-09-17T09:28:52.238 INFO:teuthology.orchestra.run.smithi078.stderr:Creating initial keys... 2024-09-17T09:28:52.648 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-authtool: AQDUS+lmYMafIRAAh+lPRBMhqlkrYPp0UmvSiA== 2024-09-17T09:28:53.654 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-authtool: AQDVS+lmqUVZIRAA2nL+rtNy8fVS/SBBIq+wLg== 2024-09-17T09:28:54.659 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-authtool: AQDWS+lmjQisIRAACjczjdZTMq7rbgp1SksfPQ== 2024-09-17T09:28:55.292 INFO:teuthology.orchestra.run.smithi078.stderr:Creating initial monmap... 2024-09-17T09:28:55.715 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-17T09:28:55.715 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-09-17T09:28:55.715 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 35c0cd22-74d7-11ef-bceb-c7b262605968 2024-09-17T09:28:55.715 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-17T09:28:56.163 INFO:teuthology.orchestra.run.smithi078.stderr:monmaptool for smithi078 [v2:172.21.15.78:3300,v1:172.21.15.78:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-17T09:28:56.163 INFO:teuthology.orchestra.run.smithi078.stderr:setting min_mon_release = octopus 2024-09-17T09:28:56.163 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: set fsid to 35c0cd22-74d7-11ef-bceb-c7b262605968 2024-09-17T09:28:56.163 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-17T09:28:56.163 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-17T09:28:56.163 INFO:teuthology.orchestra.run.smithi078.stderr:Creating mon... 2024-09-17T09:29:00.475 INFO:teuthology.orchestra.run.smithi078.stderr:create mon.smithi078 on 2024-09-17T09:29:00.776 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-17T09:29:01.108 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-35c0cd22-74d7-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-35c0cd22-74d7-11ef-bceb-c7b262605968.target. 2024-09-17T09:29:01.108 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-35c0cd22-74d7-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-35c0cd22-74d7-11ef-bceb-c7b262605968.target. 2024-09-17T09:29:01.591 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Failed to reset failed state of unit ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@mon.smithi078.service: Unit ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@mon.smithi078.service not loaded. 2024-09-17T09:29:01.598 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Created symlink /etc/systemd/system/ceph-35c0cd22-74d7-11ef-bceb-c7b262605968.target.wants/ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@mon.smithi078.service → /etc/systemd/system/ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@.service. 2024-09-17T09:29:02.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:02 smithi078 podman[23328]: 2024-09-17 09:29:02.038514268 +0000 UTC m=+0.015907624 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-17T09:29:03.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:03 smithi078 podman[23328]: 2024-09-17 09:29:03.206811741 +0000 UTC m=+1.184205094 container create abc2e65ccdc6f38b8eac31bdb36ecd9a41c0448f0ae182b3ee0cc9c8bc7c57f5 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi078, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, name=centos-stream, maintainer=Guillaume Abrioux , distribution-scope=public, version=8) 2024-09-17T09:29:04.689 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: disabled 2024-09-17T09:29:04.697 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2024-09-17T09:29:04.698 INFO:teuthology.orchestra.run.smithi078.stderr:firewalld.service is not enabled 2024-09-17T09:29:04.698 INFO:teuthology.orchestra.run.smithi078.stderr:Not possible to enable service . firewalld.service is not available 2024-09-17T09:29:04.699 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mon to start... 2024-09-17T09:29:04.699 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mon... 2024-09-17T09:29:04.903 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:04 smithi078 podman[23328]: 2024-09-17 09:29:04.597661137 +0000 UTC m=+2.575054500 container init abc2e65ccdc6f38b8eac31bdb36ecd9a41c0448f0ae182b3ee0cc9c8bc7c57f5 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi078, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , ceph=True, release=754, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, vcs-type=git, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-17T09:29:04.904 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:04 smithi078 podman[23328]: 2024-09-17 09:29:04.60161784 +0000 UTC m=+2.579011197 container start abc2e65ccdc6f38b8eac31bdb36ecd9a41c0448f0ae182b3ee0cc9c8bc7c57f5 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi078, distribution-scope=public, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, maintainer=Guillaume Abrioux , release=754, version=8, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, vcs-type=git, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8) 2024-09-17T09:29:04.904 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:04 smithi078 ceph-mon[23347]: mkfs 35c0cd22-74d7-11ef-bceb-c7b262605968 2024-09-17T09:29:04.904 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:04 smithi078 ceph-mon[23347]: mon.smithi078 is new leader, mons smithi078 in quorum (ranks 0) 2024-09-17T09:29:04.904 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:04 smithi078 bash[23328]: abc2e65ccdc6f38b8eac31bdb36ecd9a41c0448f0ae182b3ee0cc9c8bc7c57f5 2024-09-17T09:29:04.904 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:04 smithi078 systemd[1]: Started Ceph mon.smithi078 for 35c0cd22-74d7-11ef-bceb-c7b262605968. 2024-09-17T09:29:05.204 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: cluster: 2024-09-17T09:29:05.205 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: id: 35c0cd22-74d7-11ef-bceb-c7b262605968 2024-09-17T09:29:05.205 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-09-17T09:29:05.205 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-17T09:29:05.205 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: services: 2024-09-17T09:29:05.205 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi078 (age 0.555527s) 2024-09-17T09:29:05.205 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mgr: no daemons active 2024-09-17T09:29:05.205 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-09-17T09:29:05.205 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-17T09:29:05.205 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: data: 2024-09-17T09:29:05.205 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-09-17T09:29:05.205 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-09-17T09:29:05.205 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-09-17T09:29:05.205 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: pgs: 2024-09-17T09:29:05.206 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-17T09:29:05.707 INFO:teuthology.orchestra.run.smithi078.stderr:mon is available 2024-09-17T09:29:05.707 INFO:teuthology.orchestra.run.smithi078.stderr:Assimilating anything we can from ceph.conf... 2024-09-17T09:29:05.828 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:05 smithi078 ceph-mon[23347]: mon.smithi078 is new leader, mons smithi078 in quorum (ranks 0) 2024-09-17T09:29:05.828 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:05 smithi078 ceph-mon[23347]: monmap e1: 1 mons at {smithi078=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0]} 2024-09-17T09:29:05.829 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:05 smithi078 ceph-mon[23347]: fsmap 2024-09-17T09:29:05.829 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:05 smithi078 ceph-mon[23347]: osdmap e1: 0 total, 0 up, 0 in 2024-09-17T09:29:05.829 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:05 smithi078 ceph-mon[23347]: mgrmap e1: no daemons active 2024-09-17T09:29:05.829 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:05 smithi078 ceph-mon[23347]: from='client.? 172.21.15.78:0/1029931542' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-17T09:29:06.389 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-17T09:29:06.390 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [global] 2024-09-17T09:29:06.390 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: fsid = 35c0cd22-74d7-11ef-bceb-c7b262605968 2024-09-17T09:29:06.390 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.78:3300,v1:172.21.15.78:6789] 2024-09-17T09:29:06.390 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-17T09:29:06.390 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-17T09:29:06.390 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-17T09:29:06.391 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-17T09:29:06.391 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-17T09:29:06.391 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [mgr] 2024-09-17T09:29:06.391 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-17T09:29:06.391 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-17T09:29:06.391 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [osd] 2024-09-17T09:29:06.391 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-17T09:29:06.392 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-17T09:29:06.910 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:06 smithi078 ceph-mon[23347]: from='client.? 172.21.15.78:0/1999541461' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-17T09:29:06.910 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:06 smithi078 ceph-mon[23347]: from='client.? 172.21.15.78:0/1999541461' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-17T09:29:06.913 INFO:teuthology.orchestra.run.smithi078.stderr:Generating new minimal ceph.conf... 2024-09-17T09:29:08.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:08 smithi078 ceph-mon[23347]: from='client.? 172.21.15.78:0/681341296' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:29:09.120 INFO:teuthology.orchestra.run.smithi078.stderr:Restarting the monitor... 2024-09-17T09:29:09.541 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:09 smithi078 systemd[1]: Stopping Ceph mon.smithi078 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:29:09.541 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:09 smithi078 bash[23566]: Error: no container with name or ID "ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon.smithi078" found: no such container 2024-09-17T09:29:09.541 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:09 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi078[23343]: 2024-09-17T09:29:09.294+0000 7fcfd3c2a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi078 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:29:09.541 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:09 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi078[23343]: 2024-09-17T09:29:09.294+0000 7fcfd3c2a700 -1 mon.smithi078@0(leader) e1 *** Got Signal Terminated *** 2024-09-17T09:29:09.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:09 smithi078 podman[23577]: 2024-09-17 09:29:09.54033099 +0000 UTC m=+0.368209338 container died abc2e65ccdc6f38b8eac31bdb36ecd9a41c0448f0ae182b3ee0cc9c8bc7c57f5 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi078, io.openshift.expose-services=, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, ceph=True, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, name=centos-stream, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, distribution-scope=public, GIT_CLEAN=True, release=754, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-17T09:29:12.986 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:12 smithi078 podman[23577]: 2024-09-17 09:29:12.735026553 +0000 UTC m=+3.562904899 container cleanup abc2e65ccdc6f38b8eac31bdb36ecd9a41c0448f0ae182b3ee0cc9c8bc7c57f5 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi078, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, GIT_BRANCH=HEAD, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , release=754, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, ceph=True, name=centos-stream, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-09-17T09:29:12.987 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:12 smithi078 bash[23577]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi078 2024-09-17T09:29:12.987 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:12 smithi078 bash[23603]: Error: no container with name or ID "ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon.smithi078" found: no such container 2024-09-17T09:29:13.304 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:12 smithi078 podman[23589]: 2024-09-17 09:29:12.985648373 +0000 UTC m=+3.435715495 container remove abc2e65ccdc6f38b8eac31bdb36ecd9a41c0448f0ae182b3ee0cc9c8bc7c57f5 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi078, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, architecture=x86_64, ceph=True, maintainer=Guillaume Abrioux , distribution-scope=public, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, version=8, io.k8s.display-name=CentOS Stream 8, name=centos-stream, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, vcs-type=git, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=) 2024-09-17T09:29:13.304 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:12 smithi078 systemd[1]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@mon.smithi078.service: Deactivated successfully. 2024-09-17T09:29:13.304 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:12 smithi078 systemd[1]: Stopped Ceph mon.smithi078 for 35c0cd22-74d7-11ef-bceb-c7b262605968. 2024-09-17T09:29:13.305 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:12 smithi078 systemd[1]: Starting Ceph mon.smithi078 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:29:13.305 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 podman[23670]: 2024-09-17 09:29:13.203165469 +0000 UTC m=+0.099621597 container create 09299430e248d2be74bcfcf04c8960ca42faf745d77f0fb3c9294a8dc0d7f44a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi078, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., version=8, io.buildah.version=1.19.8, name=centos-stream, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, io.openshift.expose-services=, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, io.openshift.tags=base centos centos-stream, RELEASE=HEAD) 2024-09-17T09:29:13.305 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 podman[23670]: 2024-09-17 09:29:13.119915697 +0000 UTC m=+0.016371794 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-17T09:29:13.492 INFO:teuthology.orchestra.run.smithi078.stderr:Setting mon public_network to 172.21.0.0/20 2024-09-17T09:29:13.709 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 podman[23670]: 2024-09-17 09:29:13.40597606 +0000 UTC m=+0.302432147 container init 09299430e248d2be74bcfcf04c8960ca42faf745d77f0fb3c9294a8dc0d7f44a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi078, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , architecture=x86_64, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, GIT_CLEAN=True, name=centos-stream, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., io.openshift.expose-services=, RELEASE=HEAD, ceph=True, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-17T09:29:13.709 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 podman[23670]: 2024-09-17 09:29:13.409229427 +0000 UTC m=+0.305685510 container start 09299430e248d2be74bcfcf04c8960ca42faf745d77f0fb3c9294a8dc0d7f44a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi078, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, io.k8s.display-name=CentOS Stream 8, version=8, maintainer=Guillaume Abrioux , RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, release=754, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, GIT_BRANCH=HEAD) 2024-09-17T09:29:13.709 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: set uid:gid to 167:167 (ceph:ceph) 2024-09-17T09:29:13.709 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-17T09:29:13.709 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: pidfile_write: ignore empty --pid-file 2024-09-17T09:29:13.709 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: load: jerasure load: lrc 2024-09-17T09:29:13.710 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: RocksDB version: 6.15.5 2024-09-17T09:29:13.710 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-17T09:29:13.710 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Compile date Apr 18 2022 2024-09-17T09:29:13.710 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: DB SUMMARY 2024-09-17T09:29:13.710 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: DB Session ID: JRKQ2FD6SUP33VRKHZZQ 2024-09-17T09:29:13.710 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: CURRENT file: CURRENT 2024-09-17T09:29:13.710 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: IDENTITY file: IDENTITY 2024-09-17T09:29:13.710 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-17T09:29:13.710 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi078/store.db dir, Total Num: 1, files: 000008.sst 2024-09-17T09:29:13.711 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi078/store.db: 000010.log size: 82711 ; 2024-09-17T09:29:13.711 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.error_if_exists: 0 2024-09-17T09:29:13.711 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.create_if_missing: 0 2024-09-17T09:29:13.711 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.paranoid_checks: 1 2024-09-17T09:29:13.711 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-17T09:29:13.711 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.env: 0x55ea7ccc6860 2024-09-17T09:29:13.711 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.fs: Posix File System 2024-09-17T09:29:13.711 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.info_log: 0x55ea7e831fa0 2024-09-17T09:29:13.711 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.max_file_opening_threads: 16 2024-09-17T09:29:13.711 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.statistics: (nil) 2024-09-17T09:29:13.711 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.use_fsync: 0 2024-09-17T09:29:13.711 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.max_log_file_size: 0 2024-09-17T09:29:13.711 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-17T09:29:13.712 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-17T09:29:13.712 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.keep_log_file_num: 1000 2024-09-17T09:29:13.712 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.recycle_log_file_num: 0 2024-09-17T09:29:13.712 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.allow_fallocate: 1 2024-09-17T09:29:13.712 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.allow_mmap_reads: 0 2024-09-17T09:29:13.712 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.allow_mmap_writes: 0 2024-09-17T09:29:13.712 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.use_direct_reads: 0 2024-09-17T09:29:13.712 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-17T09:29:13.712 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.create_missing_column_families: 0 2024-09-17T09:29:13.712 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.db_log_dir: 2024-09-17T09:29:13.712 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi078/store.db 2024-09-17T09:29:13.712 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-17T09:29:13.712 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-17T09:29:13.713 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-17T09:29:13.713 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-17T09:29:13.713 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-17T09:29:13.713 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-17T09:29:13.713 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.advise_random_on_open: 1 2024-09-17T09:29:13.714 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.db_write_buffer_size: 0 2024-09-17T09:29:13.714 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.write_buffer_manager: 0x55ea7e922330 2024-09-17T09:29:13.714 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-17T09:29:13.714 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-17T09:29:13.715 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-17T09:29:13.715 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-17T09:29:13.715 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.rate_limiter: (nil) 2024-09-17T09:29:13.715 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-17T09:29:13.715 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.wal_recovery_mode: 2 2024-09-17T09:29:13.715 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.enable_thread_tracking: 0 2024-09-17T09:29:13.715 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.enable_pipelined_write: 0 2024-09-17T09:29:13.715 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.unordered_write: 0 2024-09-17T09:29:13.715 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-17T09:29:13.715 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-17T09:29:13.715 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-17T09:29:13.715 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-17T09:29:13.715 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.row_cache: None 2024-09-17T09:29:13.716 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.wal_filter: None 2024-09-17T09:29:13.716 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-17T09:29:13.716 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.allow_ingest_behind: 0 2024-09-17T09:29:13.716 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.preserve_deletes: 0 2024-09-17T09:29:13.716 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.two_write_queues: 0 2024-09-17T09:29:13.716 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.manual_wal_flush: 0 2024-09-17T09:29:13.716 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.atomic_flush: 0 2024-09-17T09:29:13.716 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-17T09:29:13.716 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-17T09:29:13.716 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-17T09:29:13.716 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.log_readahead_size: 0 2024-09-17T09:29:13.716 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-17T09:29:13.716 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.best_efforts_recovery: 0 2024-09-17T09:29:13.716 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-17T09:29:13.717 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-17T09:29:13.717 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.allow_data_in_errors: 0 2024-09-17T09:29:13.717 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.db_host_id: __hostname__ 2024-09-17T09:29:13.717 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.max_background_jobs: 2 2024-09-17T09:29:13.717 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.max_background_compactions: -1 2024-09-17T09:29:13.717 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.max_subcompactions: 1 2024-09-17T09:29:13.717 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-17T09:29:13.717 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-17T09:29:13.717 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-17T09:29:13.717 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.max_total_wal_size: 0 2024-09-17T09:29:13.717 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-17T09:29:13.717 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-17T09:29:13.717 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-17T09:29:13.718 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-17T09:29:13.718 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.max_open_files: -1 2024-09-17T09:29:13.718 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.bytes_per_sync: 0 2024-09-17T09:29:13.718 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-17T09:29:13.718 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-17T09:29:13.718 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.compaction_readahead_size: 0 2024-09-17T09:29:13.718 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.max_background_flushes: -1 2024-09-17T09:29:13.718 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Compression algorithms supported: 2024-09-17T09:29:13.718 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-17T09:29:13.718 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: kZSTD supported: 0 2024-09-17T09:29:13.718 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: kXpressCompression supported: 0 2024-09-17T09:29:13.718 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: kLZ4HCCompression supported: 1 2024-09-17T09:29:13.718 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: kLZ4Compression supported: 1 2024-09-17T09:29:13.719 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: kBZip2Compression supported: 0 2024-09-17T09:29:13.719 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: kZlibCompression supported: 1 2024-09-17T09:29:13.719 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: kSnappyCompression supported: 1 2024-09-17T09:29:13.719 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-17T09:29:13.719 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi078/store.db/MANIFEST-000009 2024-09-17T09:29:13.719 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-17T09:29:13.719 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-17T09:29:13.719 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.merge_operator: 2024-09-17T09:29:13.719 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.compaction_filter: None 2024-09-17T09:29:13.719 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.compaction_filter_factory: None 2024-09-17T09:29:13.719 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.sst_partitioner_factory: None 2024-09-17T09:29:13.719 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-17T09:29:13.719 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-17T09:29:13.719 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ea7e7fdd00) 2024-09-17T09:29:13.720 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: cache_index_and_filter_blocks: 1 2024-09-17T09:29:13.720 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-17T09:29:13.720 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-17T09:29:13.720 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: pin_top_level_index_and_filter: 1 2024-09-17T09:29:13.720 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: index_type: 0 2024-09-17T09:29:13.720 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: data_block_index_type: 0 2024-09-17T09:29:13.720 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: index_shortening: 1 2024-09-17T09:29:13.720 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-17T09:29:13.720 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: hash_index_allow_collision: 1 2024-09-17T09:29:13.720 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: checksum: 1 2024-09-17T09:29:13.720 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: no_block_cache: 0 2024-09-17T09:29:13.720 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache: 0x55ea7e868170 2024-09-17T09:29:13.720 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache_name: BinnedLRUCache 2024-09-17T09:29:13.720 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache_options: 2024-09-17T09:29:13.721 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: capacity : 536870912 2024-09-17T09:29:13.721 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: num_shard_bits : 4 2024-09-17T09:29:13.721 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: strict_capacity_limit : 0 2024-09-17T09:29:13.721 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: high_pri_pool_ratio: 0.000 2024-09-17T09:29:13.721 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache_compressed: (nil) 2024-09-17T09:29:13.721 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: persistent_cache: (nil) 2024-09-17T09:29:13.721 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_size: 4096 2024-09-17T09:29:13.721 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_size_deviation: 10 2024-09-17T09:29:13.721 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_restart_interval: 16 2024-09-17T09:29:13.721 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: index_block_restart_interval: 1 2024-09-17T09:29:13.721 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: metadata_block_size: 4096 2024-09-17T09:29:13.721 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: partition_filters: 0 2024-09-17T09:29:13.721 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: use_delta_encoding: 1 2024-09-17T09:29:13.722 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-17T09:29:13.722 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: whole_key_filtering: 1 2024-09-17T09:29:13.722 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: verify_compression: 0 2024-09-17T09:29:13.722 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: read_amp_bytes_per_bit: 0 2024-09-17T09:29:13.722 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: format_version: 4 2024-09-17T09:29:13.722 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: enable_index_compression: 1 2024-09-17T09:29:13.722 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_align: 0 2024-09-17T09:29:13.722 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.write_buffer_size: 33554432 2024-09-17T09:29:13.722 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.max_write_buffer_number: 2 2024-09-17T09:29:13.722 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.compression: NoCompression 2024-09-17T09:29:13.722 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.bottommost_compression: Disabled 2024-09-17T09:29:13.722 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.prefix_extractor: nullptr 2024-09-17T09:29:13.722 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-17T09:29:13.722 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.num_levels: 7 2024-09-17T09:29:13.723 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-17T09:29:13.723 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-17T09:29:13.723 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-17T09:29:13.723 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-17T09:29:13.723 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-17T09:29:13.723 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-17T09:29:13.723 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-17T09:29:13.723 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-17T09:29:13.723 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-17T09:29:13.723 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-17T09:29:13.723 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-17T09:29:13.723 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.compression_opts.level: 32767 2024-09-17T09:29:13.723 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.compression_opts.strategy: 0 2024-09-17T09:29:13.723 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-17T09:29:13.724 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-17T09:29:13.724 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-17T09:29:13.724 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.compression_opts.enabled: false 2024-09-17T09:29:13.724 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-17T09:29:13.724 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-17T09:29:13.724 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-17T09:29:13.724 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.target_file_size_base: 67108864 2024-09-17T09:29:13.724 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-17T09:29:13.724 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-17T09:29:13.724 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-17T09:29:13.724 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-17T09:29:13.724 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-17T09:29:13.724 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-17T09:29:13.725 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-17T09:29:13.725 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-17T09:29:13.725 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-17T09:29:13.725 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-17T09:29:13.725 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-17T09:29:13.725 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-17T09:29:13.725 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-17T09:29:13.725 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.arena_block_size: 4194304 2024-09-17T09:29:13.725 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-17T09:29:13.725 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-17T09:29:13.725 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-17T09:29:13.725 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.disable_auto_compactions: 0 2024-09-17T09:29:13.725 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-17T09:29:13.725 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-17T09:29:13.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-17T09:29:13.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-17T09:29:13.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-17T09:29:13.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-17T09:29:13.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-17T09:29:13.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-17T09:29:13.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-17T09:29:13.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-17T09:29:13.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.table_properties_collectors: 2024-09-17T09:29:13.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.inplace_update_support: 0 2024-09-17T09:29:13.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-17T09:29:13.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-17T09:29:13.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-17T09:29:13.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-17T09:29:13.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.bloom_locality: 0 2024-09-17T09:29:13.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.max_successive_merges: 0 2024-09-17T09:29:13.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-17T09:29:13.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.paranoid_file_checks: 0 2024-09-17T09:29:13.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.force_consistency_checks: 1 2024-09-17T09:29:13.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.report_bg_io_stats: 0 2024-09-17T09:29:13.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.ttl: 2592000 2024-09-17T09:29:13.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-17T09:29:13.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.enable_blob_files: false 2024-09-17T09:29:13.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.min_blob_size: 0 2024-09-17T09:29:13.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.blob_file_size: 268435456 2024-09-17T09:29:13.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-17T09:29:13.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-17T09:29:13.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-17T09:29:13.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi078/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-17T09:29:13.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-09-17T09:29:13.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-09-17T09:29:13.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726565353435447, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-17T09:29:13.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-09-17T09:29:13.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-17T09:29:13.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726565353436137, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79393, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77657, "index_size": 224, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9884, "raw_average_key_size": 47, "raw_value_size": 71966, "raw_average_value_size": 344, "num_data_blocks": 10, "num_entries": 209, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726565353, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "9a998869-c4bb-4bee-80b7-3026f81f1e89", "db_session_id": "JRKQ2FD6SUP33VRKHZZQ"}} 2024-09-17T09:29:13.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-09-17T09:29:13.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726565353436448, "job": 1, "event": "recovery_finished"} 2024-09-17T09:29:13.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi078/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-17T09:29:13.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55ea7e84ea80 2024-09-17T09:29:13.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: DB pointer 0x55ea7e85e000 2024-09-17T09:29:13.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-17T09:29:13.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-17T09:29:13.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** DB Stats ** 2024-09-17T09:29:13.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T09:29:13.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-17T09:29:13.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T09:29:13.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T09:29:13.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-17T09:29:13.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-17T09:29:13.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T09:29:13.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-17T09:29:13.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** Compaction Stats [default] ** 2024-09-17T09:29:13.730 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T09:29:13.730 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T09:29:13.730 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: L0 2/0 79.33 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 161.4 0.00 0.00 1 0.000 0 0 2024-09-17T09:29:13.730 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Sum 2/0 79.33 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 161.4 0.00 0.00 1 0.000 0 0 2024-09-17T09:29:13.730 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 161.4 0.00 0.00 1 0.000 0 0 2024-09-17T09:29:13.730 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-17T09:29:13.730 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** Compaction Stats [default] ** 2024-09-17T09:29:13.730 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T09:29:13.730 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T09:29:13.730 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 161.4 0.00 0.00 1 0.000 0 0 2024-09-17T09:29:13.730 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T09:29:13.730 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-17T09:29:13.730 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T09:29:13.730 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T09:29:13.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T09:29:13.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-17T09:29:13.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative compaction: 0.00 GB write, 25.20 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T09:29:13.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval compaction: 0.00 GB write, 25.20 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T09:29:13.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T09:29:13.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-17T09:29:13.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-17T09:29:13.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-17T09:29:13.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** Compaction Stats [default] ** 2024-09-17T09:29:13.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T09:29:13.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T09:29:13.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: L0 2/0 79.33 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 161.4 0.00 0.00 1 0.000 0 0 2024-09-17T09:29:13.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Sum 2/0 79.33 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 161.4 0.00 0.00 1 0.000 0 0 2024-09-17T09:29:13.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-17T09:29:13.732 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-17T09:29:13.732 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** Compaction Stats [default] ** 2024-09-17T09:29:13.732 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T09:29:13.732 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T09:29:13.732 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 161.4 0.00 0.00 1 0.000 0 0 2024-09-17T09:29:13.732 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T09:29:13.732 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-17T09:29:13.732 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T09:29:13.732 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T09:29:13.732 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T09:29:13.732 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-17T09:29:13.732 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative compaction: 0.00 GB write, 24.94 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T09:29:13.732 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T09:29:13.733 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T09:29:13.733 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-17T09:29:13.733 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-17T09:29:13.733 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: starting mon.smithi078 rank 0 at public addrs [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] at bind addrs [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi078 fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 2024-09-17T09:29:13.733 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: mon.smithi078@-1(???) e1 preinit fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 2024-09-17T09:29:13.733 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: mon.smithi078@-1(???).mds e1 new map 2024-09-17T09:29:13.733 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: mon.smithi078@-1(???).mds e1 print_map 2024-09-17T09:29:13.733 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: e1 2024-09-17T09:29:13.733 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-17T09:29:13.733 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T09:29:13.733 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: legacy client fscid: -1 2024-09-17T09:29:13.733 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-17T09:29:13.734 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: No filesystems configured 2024-09-17T09:29:13.734 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: mon.smithi078@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-17T09:29:13.734 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: mon.smithi078@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-17T09:29:13.734 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: mon.smithi078@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-17T09:29:13.734 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: mon.smithi078@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-17T09:29:13.734 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: mon.smithi078@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-17T09:29:13.734 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 bash[23670]: 09299430e248d2be74bcfcf04c8960ca42faf745d77f0fb3c9294a8dc0d7f44a 2024-09-17T09:29:13.734 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 systemd[1]: Started Ceph mon.smithi078 for 35c0cd22-74d7-11ef-bceb-c7b262605968. 2024-09-17T09:29:13.734 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: mon.smithi078 is new leader, mons smithi078 in quorum (ranks 0) 2024-09-17T09:29:13.734 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: monmap e1: 1 mons at {smithi078=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0]} 2024-09-17T09:29:13.734 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: fsmap 2024-09-17T09:29:13.734 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: osdmap e1: 0 total, 0 up, 0 in 2024-09-17T09:29:13.734 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:13 smithi078 ceph-mon[23689]: mgrmap e1: no daemons active 2024-09-17T09:29:14.465 INFO:teuthology.orchestra.run.smithi078.stderr:Wrote config to /etc/ceph/ceph.conf 2024-09-17T09:29:14.465 INFO:teuthology.orchestra.run.smithi078.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-17T09:29:14.465 INFO:teuthology.orchestra.run.smithi078.stderr:Creating mgr... 2024-09-17T09:29:14.465 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying port 9283 ... 2024-09-17T09:29:14.638 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Failed to reset failed state of unit ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@mgr.smithi078.rsqofi.service: Unit ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@mgr.smithi078.rsqofi.service not loaded. 2024-09-17T09:29:14.645 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Created symlink /etc/systemd/system/ceph-35c0cd22-74d7-11ef-bceb-c7b262605968.target.wants/ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@mgr.smithi078.rsqofi.service → /etc/systemd/system/ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@.service. 2024-09-17T09:29:15.298 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:14 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/200194662' entity='client.admin' 2024-09-17T09:29:15.311 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: disabled 2024-09-17T09:29:15.320 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2024-09-17T09:29:15.321 INFO:teuthology.orchestra.run.smithi078.stderr:firewalld.service is not enabled 2024-09-17T09:29:15.322 INFO:teuthology.orchestra.run.smithi078.stderr:Not possible to enable service . firewalld.service is not available 2024-09-17T09:29:15.329 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: disabled 2024-09-17T09:29:15.336 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2024-09-17T09:29:15.337 INFO:teuthology.orchestra.run.smithi078.stderr:firewalld.service is not enabled 2024-09-17T09:29:15.337 INFO:teuthology.orchestra.run.smithi078.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-09-17T09:29:15.337 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mgr to start... 2024-09-17T09:29:15.337 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mgr... 2024-09-17T09:29:15.888 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-17T09:29:15.888 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-09-17T09:29:15.889 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsid": "35c0cd22-74d7-11ef-bceb-c7b262605968", 2024-09-17T09:29:15.889 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "health": { 2024-09-17T09:29:15.889 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-17T09:29:15.889 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "checks": {}, 2024-09-17T09:29:15.889 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mutes": [] 2024-09-17T09:29:15.889 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-17T09:29:15.889 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-17T09:29:15.889 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum": [ 2024-09-17T09:29:15.889 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 0 2024-09-17T09:29:15.889 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-17T09:29:15.889 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-17T09:29:15.889 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "smithi078" 2024-09-17T09:29:15.889 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-17T09:29:15.890 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-09-17T09:29:15.890 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "monmap": { 2024-09-17T09:29:15.890 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T09:29:15.890 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-17T09:29:15.890 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-17T09:29:15.890 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-17T09:29:15.890 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osdmap": { 2024-09-17T09:29:15.890 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T09:29:15.890 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-17T09:29:15.890 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-17T09:29:15.890 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-17T09:29:15.890 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-17T09:29:15.893 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-17T09:29:15.894 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-17T09:29:15.894 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-17T09:29:15.894 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgmap": { 2024-09-17T09:29:15.894 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-17T09:29:15.894 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-17T09:29:15.894 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-17T09:29:15.894 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-17T09:29:15.894 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-17T09:29:15.894 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-17T09:29:15.894 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-17T09:29:15.894 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-17T09:29:15.895 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-17T09:29:15.895 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsmap": { 2024-09-17T09:29:15.895 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T09:29:15.895 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-17T09:29:15.895 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-17T09:29:15.895 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-17T09:29:15.895 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-17T09:29:15.895 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": false, 2024-09-17T09:29:15.895 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-17T09:29:15.895 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modules": [ 2024-09-17T09:29:15.895 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "iostat", 2024-09-17T09:29:15.895 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "nfs", 2024-09-17T09:29:15.895 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "restful" 2024-09-17T09:29:15.895 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-17T09:29:15.896 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-09-17T09:29:15.896 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-17T09:29:15.896 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "servicemap": { 2024-09-17T09:29:15.896 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T09:29:15.896 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modified": "2024-09-17T09:29:04.648062+0000", 2024-09-17T09:29:15.896 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-09-17T09:29:15.896 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-17T09:29:15.896 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-17T09:29:15.896 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-09-17T09:29:16.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:15 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/823391720' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-17T09:29:16.482 INFO:teuthology.orchestra.run.smithi078.stderr:mgr not available, waiting (1/15)... 2024-09-17T09:29:19.037 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-17T09:29:19.037 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-09-17T09:29:19.037 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsid": "35c0cd22-74d7-11ef-bceb-c7b262605968", 2024-09-17T09:29:19.037 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "health": { 2024-09-17T09:29:19.037 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-17T09:29:19.038 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "checks": {}, 2024-09-17T09:29:19.038 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mutes": [] 2024-09-17T09:29:19.038 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-17T09:29:19.038 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-17T09:29:19.038 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum": [ 2024-09-17T09:29:19.038 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 0 2024-09-17T09:29:19.038 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-17T09:29:19.038 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-17T09:29:19.038 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "smithi078" 2024-09-17T09:29:19.038 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-17T09:29:19.038 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_age": 5, 2024-09-17T09:29:19.038 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "monmap": { 2024-09-17T09:29:19.038 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T09:29:19.039 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-17T09:29:19.039 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-17T09:29:19.039 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-17T09:29:19.039 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osdmap": { 2024-09-17T09:29:19.039 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T09:29:19.039 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-17T09:29:19.039 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-17T09:29:19.039 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-17T09:29:19.039 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-17T09:29:19.039 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-17T09:29:19.039 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-17T09:29:19.039 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-17T09:29:19.039 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgmap": { 2024-09-17T09:29:19.040 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-17T09:29:19.040 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-17T09:29:19.040 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-17T09:29:19.040 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-17T09:29:19.040 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-17T09:29:19.040 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-17T09:29:19.040 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-17T09:29:19.040 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-17T09:29:19.040 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-17T09:29:19.040 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsmap": { 2024-09-17T09:29:19.040 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T09:29:19.040 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-17T09:29:19.040 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-17T09:29:19.041 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-17T09:29:19.041 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-17T09:29:19.041 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": false, 2024-09-17T09:29:19.041 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-17T09:29:19.041 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modules": [ 2024-09-17T09:29:19.041 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "iostat", 2024-09-17T09:29:19.041 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "nfs", 2024-09-17T09:29:19.041 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "restful" 2024-09-17T09:29:19.041 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-17T09:29:19.041 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-09-17T09:29:19.041 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-17T09:29:19.041 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "servicemap": { 2024-09-17T09:29:19.041 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T09:29:19.041 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modified": "2024-09-17T09:29:04.648062+0000", 2024-09-17T09:29:19.042 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-09-17T09:29:19.042 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-17T09:29:19.042 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-17T09:29:19.042 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-09-17T09:29:19.669 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:19 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/2641880970' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-17T09:29:19.672 INFO:teuthology.orchestra.run.smithi078.stderr:mgr not available, waiting (2/15)... 2024-09-17T09:29:21.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:21 smithi078 ceph-mon[23689]: Activating manager daemon smithi078.rsqofi 2024-09-17T09:29:21.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:21 smithi078 ceph-mon[23689]: mgrmap e2: smithi078.rsqofi(active, starting, since 0.00289524s) 2024-09-17T09:29:21.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:21 smithi078 ceph-mon[23689]: from='mgr.14100 172.21.15.78:0/4165039130' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T09:29:21.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:21 smithi078 ceph-mon[23689]: from='mgr.14100 172.21.15.78:0/4165039130' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T09:29:21.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:21 smithi078 ceph-mon[23689]: from='mgr.14100 172.21.15.78:0/4165039130' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T09:29:21.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:21 smithi078 ceph-mon[23689]: from='mgr.14100 172.21.15.78:0/4165039130' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-17T09:29:21.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:21 smithi078 ceph-mon[23689]: from='mgr.14100 172.21.15.78:0/4165039130' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mgr metadata", "who": "smithi078.rsqofi", "id": "smithi078.rsqofi"}]: dispatch 2024-09-17T09:29:21.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:21 smithi078 ceph-mon[23689]: Manager daemon smithi078.rsqofi is now available 2024-09-17T09:29:21.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:21 smithi078 ceph-mon[23689]: from='mgr.14100 172.21.15.78:0/4165039130' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:29:21.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:21 smithi078 ceph-mon[23689]: from='mgr.14100 172.21.15.78:0/4165039130' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/trash_purge_schedule"}]: dispatch 2024-09-17T09:29:21.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:21 smithi078 ceph-mon[23689]: from='mgr.14100 172.21.15.78:0/4165039130' entity='mgr.smithi078.rsqofi' 2024-09-17T09:29:21.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:21 smithi078 ceph-mon[23689]: from='mgr.14100 172.21.15.78:0/4165039130' entity='mgr.smithi078.rsqofi' 2024-09-17T09:29:21.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:21 smithi078 ceph-mon[23689]: from='mgr.14100 172.21.15.78:0/4165039130' entity='mgr.smithi078.rsqofi' 2024-09-17T09:29:22.223 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-17T09:29:22.223 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-09-17T09:29:22.223 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsid": "35c0cd22-74d7-11ef-bceb-c7b262605968", 2024-09-17T09:29:22.223 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "health": { 2024-09-17T09:29:22.223 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-17T09:29:22.223 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "checks": {}, 2024-09-17T09:29:22.224 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mutes": [] 2024-09-17T09:29:22.224 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-17T09:29:22.224 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-17T09:29:22.224 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum": [ 2024-09-17T09:29:22.224 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 0 2024-09-17T09:29:22.224 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-17T09:29:22.224 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-17T09:29:22.224 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "smithi078" 2024-09-17T09:29:22.224 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-17T09:29:22.224 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_age": 8, 2024-09-17T09:29:22.224 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "monmap": { 2024-09-17T09:29:22.224 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T09:29:22.224 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-17T09:29:22.225 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-17T09:29:22.225 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-17T09:29:22.225 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osdmap": { 2024-09-17T09:29:22.225 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T09:29:22.225 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-17T09:29:22.225 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-17T09:29:22.225 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-17T09:29:22.225 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-17T09:29:22.228 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-17T09:29:22.228 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-17T09:29:22.228 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-17T09:29:22.228 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgmap": { 2024-09-17T09:29:22.228 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-17T09:29:22.228 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-17T09:29:22.229 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-17T09:29:22.229 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-17T09:29:22.229 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-17T09:29:22.229 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-17T09:29:22.229 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-17T09:29:22.229 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-17T09:29:22.229 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-17T09:29:22.229 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsmap": { 2024-09-17T09:29:22.229 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T09:29:22.229 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-17T09:29:22.229 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-17T09:29:22.229 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-17T09:29:22.229 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-17T09:29:22.229 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": false, 2024-09-17T09:29:22.230 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-17T09:29:22.230 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modules": [ 2024-09-17T09:29:22.230 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "iostat", 2024-09-17T09:29:22.230 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "nfs", 2024-09-17T09:29:22.230 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "restful" 2024-09-17T09:29:22.230 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-17T09:29:22.230 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-09-17T09:29:22.230 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-17T09:29:22.230 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "servicemap": { 2024-09-17T09:29:22.230 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T09:29:22.230 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modified": "2024-09-17T09:29:04.648062+0000", 2024-09-17T09:29:22.230 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-09-17T09:29:22.230 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-17T09:29:22.231 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-17T09:29:22.231 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-09-17T09:29:22.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:22 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/1155669753' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-17T09:29:22.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:22 smithi078 ceph-mon[23689]: mgrmap e3: smithi078.rsqofi(active, since 1.00619s) 2024-09-17T09:29:22.711 INFO:teuthology.orchestra.run.smithi078.stderr:mgr not available, waiting (3/15)... 2024-09-17T09:29:24.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:24 smithi078 ceph-mon[23689]: mgrmap e4: smithi078.rsqofi(active, since 2s) 2024-09-17T09:29:25.409 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-17T09:29:25.410 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-09-17T09:29:25.410 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsid": "35c0cd22-74d7-11ef-bceb-c7b262605968", 2024-09-17T09:29:25.410 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "health": { 2024-09-17T09:29:25.410 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-17T09:29:25.410 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "checks": {}, 2024-09-17T09:29:25.410 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mutes": [] 2024-09-17T09:29:25.410 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-17T09:29:25.410 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-17T09:29:25.410 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum": [ 2024-09-17T09:29:25.410 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 0 2024-09-17T09:29:25.410 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-17T09:29:25.410 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-17T09:29:25.411 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "smithi078" 2024-09-17T09:29:25.411 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-17T09:29:25.411 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_age": 11, 2024-09-17T09:29:25.411 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "monmap": { 2024-09-17T09:29:25.411 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T09:29:25.411 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-17T09:29:25.411 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-17T09:29:25.411 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-17T09:29:25.411 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osdmap": { 2024-09-17T09:29:25.411 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T09:29:25.411 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-17T09:29:25.411 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-17T09:29:25.411 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-17T09:29:25.411 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-17T09:29:25.414 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-17T09:29:25.415 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-17T09:29:25.415 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-17T09:29:25.415 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgmap": { 2024-09-17T09:29:25.415 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-17T09:29:25.415 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-17T09:29:25.415 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-17T09:29:25.415 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-17T09:29:25.415 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-17T09:29:25.415 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-17T09:29:25.415 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-17T09:29:25.415 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-17T09:29:25.415 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-17T09:29:25.415 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsmap": { 2024-09-17T09:29:25.415 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T09:29:25.416 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-17T09:29:25.416 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-17T09:29:25.416 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-17T09:29:25.416 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-17T09:29:25.416 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": true, 2024-09-17T09:29:25.416 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-17T09:29:25.416 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modules": [ 2024-09-17T09:29:25.416 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "iostat", 2024-09-17T09:29:25.416 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "nfs", 2024-09-17T09:29:25.416 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "restful" 2024-09-17T09:29:25.416 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-17T09:29:25.416 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-09-17T09:29:25.416 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-17T09:29:25.416 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "servicemap": { 2024-09-17T09:29:25.417 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T09:29:25.417 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modified": "2024-09-17T09:29:04.648062+0000", 2024-09-17T09:29:25.417 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-09-17T09:29:25.417 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-17T09:29:25.417 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-17T09:29:25.417 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-09-17T09:29:25.845 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:25 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/2652439431' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-17T09:29:25.976 INFO:teuthology.orchestra.run.smithi078.stderr:mgr is available 2024-09-17T09:29:26.590 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-17T09:29:26.590 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [global] 2024-09-17T09:29:26.590 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: fsid = 35c0cd22-74d7-11ef-bceb-c7b262605968 2024-09-17T09:29:26.590 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-17T09:29:26.590 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-17T09:29:26.590 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-17T09:29:26.590 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-17T09:29:26.591 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-17T09:29:26.591 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [mgr] 2024-09-17T09:29:26.591 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-17T09:29:26.591 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-17T09:29:26.591 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [osd] 2024-09-17T09:29:26.591 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-17T09:29:26.591 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-17T09:29:26.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:26 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/453700884' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-17T09:29:27.132 INFO:teuthology.orchestra.run.smithi078.stderr:Enabling cephadm module... 2024-09-17T09:29:28.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:27 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/2352898737' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-17T09:29:29.006 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:28 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/2352898737' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-17T09:29:29.006 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:28 smithi078 ceph-mon[23689]: mgrmap e5: smithi078.rsqofi(active, since 6s) 2024-09-17T09:29:29.089 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-09-17T09:29:29.089 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 5, 2024-09-17T09:29:29.089 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": true, 2024-09-17T09:29:29.089 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "active_name": "smithi078.rsqofi", 2024-09-17T09:29:29.089 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-17T09:29:29.089 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-09-17T09:29:29.653 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for the mgr to restart... 2024-09-17T09:29:29.653 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mgr epoch 5... 2024-09-17T09:29:29.873 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:29 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/566453181' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-17T09:29:34.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:34 smithi078 ceph-mon[23689]: Active manager daemon smithi078.rsqofi restarted 2024-09-17T09:29:34.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:34 smithi078 ceph-mon[23689]: Activating manager daemon smithi078.rsqofi 2024-09-17T09:29:34.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:34 smithi078 ceph-mon[23689]: osdmap e2: 0 total, 0 up, 0 in 2024-09-17T09:29:35.460 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-09-17T09:29:35.460 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-09-17T09:29:35.460 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "initialized": true 2024-09-17T09:29:35.460 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-09-17T09:29:35.784 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:35 smithi078 ceph-mon[23689]: mgrmap e6: smithi078.rsqofi(active, starting, since 0.483539s) 2024-09-17T09:29:35.785 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:35 smithi078 ceph-mon[23689]: from='mgr.14120 172.21.15.78:0/3290202472' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-17T09:29:35.785 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:35 smithi078 ceph-mon[23689]: from='mgr.14120 172.21.15.78:0/3290202472' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mgr metadata", "who": "smithi078.rsqofi", "id": "smithi078.rsqofi"}]: dispatch 2024-09-17T09:29:35.785 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:35 smithi078 ceph-mon[23689]: from='mgr.14120 172.21.15.78:0/3290202472' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T09:29:35.785 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:35 smithi078 ceph-mon[23689]: from='mgr.14120 172.21.15.78:0/3290202472' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T09:29:35.785 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:35 smithi078 ceph-mon[23689]: from='mgr.14120 172.21.15.78:0/3290202472' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T09:29:35.785 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:35 smithi078 ceph-mon[23689]: Manager daemon smithi078.rsqofi is now available 2024-09-17T09:29:35.785 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:35 smithi078 ceph-mon[23689]: from='mgr.14120 172.21.15.78:0/3290202472' entity='mgr.smithi078.rsqofi' 2024-09-17T09:29:35.785 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:35 smithi078 ceph-mon[23689]: from='mgr.14120 172.21.15.78:0/3290202472' entity='mgr.smithi078.rsqofi' 2024-09-17T09:29:35.785 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:35 smithi078 ceph-mon[23689]: from='mgr.14120 172.21.15.78:0/3290202472' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:29:35.785 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:35 smithi078 ceph-mon[23689]: from='mgr.14120 172.21.15.78:0/3290202472' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:29:35.785 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:35 smithi078 ceph-mon[23689]: from='mgr.14120 172.21.15.78:0/3290202472' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:29:35.785 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:35 smithi078 ceph-mon[23689]: from='mgr.14120 172.21.15.78:0/3290202472' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:29:35.785 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:35 smithi078 ceph-mon[23689]: from='mgr.14120 172.21.15.78:0/3290202472' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/trash_purge_schedule"}]: dispatch 2024-09-17T09:29:36.067 INFO:teuthology.orchestra.run.smithi078.stderr:mgr epoch 5 is available 2024-09-17T09:29:36.067 INFO:teuthology.orchestra.run.smithi078.stderr:Setting orchestrator backend to cephadm... 2024-09-17T09:29:36.704 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:36 smithi078 ceph-mon[23689]: mgrmap e7: smithi078.rsqofi(active, since 1.48775s) 2024-09-17T09:29:36.704 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:36 smithi078 ceph-mon[23689]: [17/Sep/2024:09:29:35] ENGINE Bus STARTING 2024-09-17T09:29:36.705 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:36 smithi078 ceph-mon[23689]: [17/Sep/2024:09:29:35] ENGINE Serving on https://172.21.15.78:7150 2024-09-17T09:29:36.705 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:36 smithi078 ceph-mon[23689]: [17/Sep/2024:09:29:35] ENGINE Bus STARTED 2024-09-17T09:29:36.705 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:36 smithi078 ceph-mon[23689]: from='mgr.14120 172.21.15.78:0/3290202472' entity='mgr.smithi078.rsqofi' 2024-09-17T09:29:36.705 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:36 smithi078 ceph-mon[23689]: from='mgr.14120 172.21.15.78:0/3290202472' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:29:37.869 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: value unchanged 2024-09-17T09:29:38.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:37 smithi078 ceph-mon[23689]: mgrmap e8: smithi078.rsqofi(active, since 2s) 2024-09-17T09:29:38.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:37 smithi078 ceph-mon[23689]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:29:38.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:37 smithi078 ceph-mon[23689]: from='mgr.14120 172.21.15.78:0/3290202472' entity='mgr.smithi078.rsqofi' 2024-09-17T09:29:38.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:37 smithi078 ceph-mon[23689]: from='mgr.14120 172.21.15.78:0/3290202472' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:29:38.462 INFO:teuthology.orchestra.run.smithi078.stderr:Generating ssh key... 2024-09-17T09:29:39.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:39 smithi078 ceph-mon[23689]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:29:39.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:39 smithi078 ceph-mon[23689]: from='mgr.14120 172.21.15.78:0/3290202472' entity='mgr.smithi078.rsqofi' 2024-09-17T09:29:39.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:39 smithi078 ceph-mon[23689]: from='mgr.14120 172.21.15.78:0/3290202472' entity='mgr.smithi078.rsqofi' 2024-09-17T09:29:40.465 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC1e+zPEbqBK+IH+X00l5Dwdao98tnmRymQ0tRxmhtBtE536A0prON1FXEWbLv+xxPQQnJuOLMAExLSewt1GUphRjkTATNSoZjkP6p+lr1K57KRONe0mmZAbNZjvIlqlutq4CJWddDPOFREpRT81EHMIUs0y7/wcdJeLcHK6ygqObV67ooXk2IC+zFuyX8c52asrcLx62eW0G7UldySMIresEAwU8IKHsU+pKtVFwmyVri32oj/xhlV53VxoAu9SX3nRvQH5W0T7CdOs/it4JtAHvMvx8vum8bMf0cqkyAsDPsp5PgwMtLElRr8YYwcRAh0Exb3WQuf9PnCZUa8J7/w/Dd/96tNjdbfhwY3cKr4CiyehYnN96aWsGyXu66TNxndAxobQvb/k53FwiF7RqobsTAncF5vPFywCIIg2HwbhbSD1jXlnEwgDc1P/LhWFHdLjAZysYCA2UQ4uUiyL8aZxwkfQYgnxC+7LDMIZgVhey4wpvFxbMbAL4aURv0rbFk= ceph-35c0cd22-74d7-11ef-bceb-c7b262605968 2024-09-17T09:29:40.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:40 smithi078 ceph-mon[23689]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:29:40.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:40 smithi078 ceph-mon[23689]: Generating ssh key... 2024-09-17T09:29:41.084 INFO:teuthology.orchestra.run.smithi078.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-17T09:29:41.084 INFO:teuthology.orchestra.run.smithi078.stderr:Adding key to root@localhost authorized_keys... 2024-09-17T09:29:41.090 INFO:teuthology.orchestra.run.smithi078.stderr:Adding host smithi078... 2024-09-17T09:29:41.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:41 smithi078 ceph-mon[23689]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:29:42.619 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Added host 'smithi078' with addr '172.21.15.78' 2024-09-17T09:29:42.659 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:42 smithi078 ceph-mon[23689]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi078", "addr": "172.21.15.78", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:29:43.215 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying mon service with default placement... 2024-09-17T09:29:43.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:43 smithi078 ceph-mon[23689]: Deploying cephadm binary to smithi078 2024-09-17T09:29:43.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:43 smithi078 ceph-mon[23689]: from='mgr.14120 172.21.15.78:0/3290202472' entity='mgr.smithi078.rsqofi' 2024-09-17T09:29:43.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:43 smithi078 ceph-mon[23689]: Added host smithi078 2024-09-17T09:29:43.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:43 smithi078 ceph-mon[23689]: from='mgr.14120 172.21.15.78:0/3290202472' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:29:43.853 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled mon update... 2024-09-17T09:29:44.749 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying mgr service with default placement... 2024-09-17T09:29:45.024 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:44 smithi078 ceph-mon[23689]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:29:45.024 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:44 smithi078 ceph-mon[23689]: Saving service mon spec with placement count:5 2024-09-17T09:29:45.024 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:44 smithi078 ceph-mon[23689]: from='mgr.14120 172.21.15.78:0/3290202472' entity='mgr.smithi078.rsqofi' 2024-09-17T09:29:45.319 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-09-17T09:29:46.292 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying crash service with default placement... 2024-09-17T09:29:46.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:46 smithi078 ceph-mon[23689]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:29:46.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:46 smithi078 ceph-mon[23689]: Saving service mgr spec with placement count:2 2024-09-17T09:29:46.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:46 smithi078 ceph-mon[23689]: from='mgr.14120 172.21.15.78:0/3290202472' entity='mgr.smithi078.rsqofi' 2024-09-17T09:29:47.468 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled crash update... 2024-09-17T09:29:48.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:47 smithi078 ceph-mon[23689]: from='mgr.14120 172.21.15.78:0/3290202472' entity='mgr.smithi078.rsqofi' 2024-09-17T09:29:48.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:47 smithi078 ceph-mon[23689]: from='mgr.14120 172.21.15.78:0/3290202472' entity='mgr.smithi078.rsqofi' 2024-09-17T09:29:48.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:47 smithi078 ceph-mon[23689]: from='mgr.14120 172.21.15.78:0/3290202472' entity='mgr.smithi078.rsqofi' 2024-09-17T09:29:48.470 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying prometheus service with default placement... 2024-09-17T09:29:49.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:48 smithi078 ceph-mon[23689]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:29:49.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:48 smithi078 ceph-mon[23689]: Saving service crash spec with placement * 2024-09-17T09:29:49.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:48 smithi078 ceph-mon[23689]: from='mgr.14120 172.21.15.78:0/3290202472' entity='mgr.smithi078.rsqofi' 2024-09-17T09:29:49.147 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-09-17T09:29:50.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:49 smithi078 ceph-mon[23689]: from='mgr.14120 172.21.15.78:0/3290202472' entity='mgr.smithi078.rsqofi' 2024-09-17T09:29:50.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:49 smithi078 ceph-mon[23689]: from='mgr.14120 172.21.15.78:0/3290202472' entity='mgr.smithi078.rsqofi' 2024-09-17T09:29:50.708 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying grafana service with default placement... 2024-09-17T09:29:51.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:50 smithi078 ceph-mon[23689]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:29:51.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:50 smithi078 ceph-mon[23689]: Saving service prometheus spec with placement count:1 2024-09-17T09:29:51.375 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-09-17T09:29:51.974 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying node-exporter service with default placement... 2024-09-17T09:29:52.569 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:52 smithi078 ceph-mon[23689]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:29:52.569 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:52 smithi078 ceph-mon[23689]: Saving service grafana spec with placement count:1 2024-09-17T09:29:52.569 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:52 smithi078 ceph-mon[23689]: from='mgr.14120 172.21.15.78:0/3290202472' entity='mgr.smithi078.rsqofi' 2024-09-17T09:29:52.569 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-09-17T09:29:53.196 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying alertmanager service with default placement... 2024-09-17T09:29:53.801 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:53 smithi078 ceph-mon[23689]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:29:53.801 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:53 smithi078 ceph-mon[23689]: Saving service node-exporter spec with placement * 2024-09-17T09:29:53.801 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:53 smithi078 ceph-mon[23689]: from='mgr.14120 172.21.15.78:0/3290202472' entity='mgr.smithi078.rsqofi' 2024-09-17T09:29:53.802 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-09-17T09:29:55.028 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:54 smithi078 ceph-mon[23689]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:29:55.028 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:54 smithi078 ceph-mon[23689]: Saving service alertmanager spec with placement count:1 2024-09-17T09:29:55.028 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:54 smithi078 ceph-mon[23689]: from='mgr.14120 172.21.15.78:0/3290202472' entity='mgr.smithi078.rsqofi' 2024-09-17T09:29:55.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:55 smithi078 ceph-mon[23689]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:29:55.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:55 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/904663303' entity='client.admin' 2024-09-17T09:29:56.809 INFO:teuthology.orchestra.run.smithi078.stderr:Enabling the dashboard module... 2024-09-17T09:29:57.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:57 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/309570805' entity='client.admin' 2024-09-17T09:29:57.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:57 smithi078 ceph-mon[23689]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:29:58.434 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:58 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/1011222889' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-17T09:29:59.315 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:59 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/1011222889' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-17T09:29:59.315 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:29:59 smithi078 ceph-mon[23689]: mgrmap e9: smithi078.rsqofi(active, since 24s) 2024-09-17T09:29:59.515 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-09-17T09:29:59.515 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 9, 2024-09-17T09:29:59.515 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": true, 2024-09-17T09:29:59.515 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "active_name": "smithi078.rsqofi", 2024-09-17T09:29:59.515 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-17T09:29:59.516 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-09-17T09:30:00.212 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for the mgr to restart... 2024-09-17T09:30:00.212 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mgr epoch 9... 2024-09-17T09:30:00.220 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:00 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/3863154841' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-17T09:30:04.507 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:04 smithi078 ceph-mon[23689]: Active manager daemon smithi078.rsqofi restarted 2024-09-17T09:30:04.507 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:04 smithi078 ceph-mon[23689]: Activating manager daemon smithi078.rsqofi 2024-09-17T09:30:04.507 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:04 smithi078 ceph-mon[23689]: osdmap e3: 0 total, 0 up, 0 in 2024-09-17T09:30:05.468 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-09-17T09:30:05.469 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-09-17T09:30:05.469 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "initialized": true 2024-09-17T09:30:05.469 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-09-17T09:30:05.672 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:05 smithi078 ceph-mon[23689]: mgrmap e10: smithi078.rsqofi(active, starting, since 0.214284s) 2024-09-17T09:30:05.672 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:05 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-17T09:30:05.672 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:05 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mgr metadata", "who": "smithi078.rsqofi", "id": "smithi078.rsqofi"}]: dispatch 2024-09-17T09:30:05.672 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:05 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T09:30:05.672 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:05 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T09:30:05.672 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:05 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T09:30:05.672 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:05 smithi078 ceph-mon[23689]: Manager daemon smithi078.rsqofi is now available 2024-09-17T09:30:05.672 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:05 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:30:05.672 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:05 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:30:05.672 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:05 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:30:05.673 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:05 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/trash_purge_schedule"}]: dispatch 2024-09-17T09:30:06.264 INFO:teuthology.orchestra.run.smithi078.stderr:mgr epoch 9 is available 2024-09-17T09:30:06.264 INFO:teuthology.orchestra.run.smithi078.stderr:Generating a dashboard self-signed certificate... 2024-09-17T09:30:06.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:06 smithi078 ceph-mon[23689]: [17/Sep/2024:09:30:05] ENGINE Bus STARTING 2024-09-17T09:30:06.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:06 smithi078 ceph-mon[23689]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-17T09:30:06.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:06 smithi078 ceph-mon[23689]: mgrmap e11: smithi078.rsqofi(active, since 1.21708s) 2024-09-17T09:30:06.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:06 smithi078 ceph-mon[23689]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-17T09:30:06.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:06 smithi078 ceph-mon[23689]: [17/Sep/2024:09:30:05] ENGINE Serving on https://172.21.15.78:7150 2024-09-17T09:30:06.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:06 smithi078 ceph-mon[23689]: [17/Sep/2024:09:30:05] ENGINE Bus STARTED 2024-09-17T09:30:06.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:06 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' 2024-09-17T09:30:06.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:06 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' 2024-09-17T09:30:07.114 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Self-signed certificate created 2024-09-17T09:30:07.785 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:07 smithi078 ceph-mon[23689]: mgrmap e12: smithi078.rsqofi(active, since 2s) 2024-09-17T09:30:07.785 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:07 smithi078 ceph-mon[23689]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:30:07.785 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:07 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' 2024-09-17T09:30:07.785 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:07 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' 2024-09-17T09:30:07.785 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:07 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' 2024-09-17T09:30:07.787 INFO:teuthology.orchestra.run.smithi078.stderr:Creating initial admin user... 2024-09-17T09:30:08.663 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$5.UdOFeZzZfMB1swHQs9H.PnLQwrvalvtNCMmyVeCOUolUq0AafXK", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726565408, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-17T09:30:09.522 INFO:teuthology.orchestra.run.smithi078.stderr:Fetching dashboard port number... 2024-09-17T09:30:10.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:09 smithi078 ceph-mon[23689]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:30:10.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:09 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' 2024-09-17T09:30:10.107 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 8443 2024-09-17T09:30:11.053 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:10 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/952256001' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-17T09:30:11.066 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: disabled 2024-09-17T09:30:11.073 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2024-09-17T09:30:11.074 INFO:teuthology.orchestra.run.smithi078.stderr:firewalld.service is not enabled 2024-09-17T09:30:11.074 INFO:teuthology.orchestra.run.smithi078.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-17T09:30:11.076 INFO:teuthology.orchestra.run.smithi078.stderr:Ceph Dashboard is now available at: 2024-09-17T09:30:11.076 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-17T09:30:11.076 INFO:teuthology.orchestra.run.smithi078.stderr: URL: https://smithi078.front.sepia.ceph.com:8443/ 2024-09-17T09:30:11.077 INFO:teuthology.orchestra.run.smithi078.stderr: User: admin 2024-09-17T09:30:11.077 INFO:teuthology.orchestra.run.smithi078.stderr: Password: 37bd2j2sth 2024-09-17T09:30:11.077 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-17T09:30:11.077 INFO:teuthology.orchestra.run.smithi078.stderr:Enabling autotune for osd_memory_target 2024-09-17T09:30:12.009 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:11 smithi078 ceph-mon[23689]: mgrmap e13: smithi078.rsqofi(active, since 6s) 2024-09-17T09:30:13.062 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-09-17T09:30:13.618 INFO:teuthology.orchestra.run.smithi078.stderr:You can access the Ceph CLI with: 2024-09-17T09:30:13.618 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-17T09:30:13.618 INFO:teuthology.orchestra.run.smithi078.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-17T09:30:13.618 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-17T09:30:13.618 INFO:teuthology.orchestra.run.smithi078.stderr:Please consider enabling telemetry to help improve Ceph: 2024-09-17T09:30:13.618 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-17T09:30:13.618 INFO:teuthology.orchestra.run.smithi078.stderr: ceph telemetry on 2024-09-17T09:30:13.618 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-17T09:30:13.618 INFO:teuthology.orchestra.run.smithi078.stderr:For more information see: 2024-09-17T09:30:13.618 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-17T09:30:13.618 INFO:teuthology.orchestra.run.smithi078.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-17T09:30:13.619 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-17T09:30:13.619 INFO:teuthology.orchestra.run.smithi078.stderr:Bootstrap complete. 2024-09-17T09:30:13.653 INFO:tasks.cephadm:Fetching config... 2024-09-17T09:30:13.653 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-17T09:30:13.653 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-17T09:30:13.669 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-17T09:30:13.669 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-17T09:30:13.669 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-17T09:30:13.724 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-17T09:30:13.725 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-17T09:30:13.725 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/mon.smithi078/keyring of=/dev/stdout 2024-09-17T09:30:13.791 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-17T09:30:13.791 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-17T09:30:13.791 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-17T09:30:13.846 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-17T09:30:13.847 DEBUG:teuthology.orchestra.run.smithi078:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC1e+zPEbqBK+IH+X00l5Dwdao98tnmRymQ0tRxmhtBtE536A0prON1FXEWbLv+xxPQQnJuOLMAExLSewt1GUphRjkTATNSoZjkP6p+lr1K57KRONe0mmZAbNZjvIlqlutq4CJWddDPOFREpRT81EHMIUs0y7/wcdJeLcHK6ygqObV67ooXk2IC+zFuyX8c52asrcLx62eW0G7UldySMIresEAwU8IKHsU+pKtVFwmyVri32oj/xhlV53VxoAu9SX3nRvQH5W0T7CdOs/it4JtAHvMvx8vum8bMf0cqkyAsDPsp5PgwMtLElRr8YYwcRAh0Exb3WQuf9PnCZUa8J7/w/Dd/96tNjdbfhwY3cKr4CiyehYnN96aWsGyXu66TNxndAxobQvb/k53FwiF7RqobsTAncF5vPFywCIIg2HwbhbSD1jXlnEwgDc1P/LhWFHdLjAZysYCA2UQ4uUiyL8aZxwkfQYgnxC+7LDMIZgVhey4wpvFxbMbAL4aURv0rbFk= ceph-35c0cd22-74d7-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-17T09:30:13.925 INFO:teuthology.orchestra.run.smithi078.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC1e+zPEbqBK+IH+X00l5Dwdao98tnmRymQ0tRxmhtBtE536A0prON1FXEWbLv+xxPQQnJuOLMAExLSewt1GUphRjkTATNSoZjkP6p+lr1K57KRONe0mmZAbNZjvIlqlutq4CJWddDPOFREpRT81EHMIUs0y7/wcdJeLcHK6ygqObV67ooXk2IC+zFuyX8c52asrcLx62eW0G7UldySMIresEAwU8IKHsU+pKtVFwmyVri32oj/xhlV53VxoAu9SX3nRvQH5W0T7CdOs/it4JtAHvMvx8vum8bMf0cqkyAsDPsp5PgwMtLElRr8YYwcRAh0Exb3WQuf9PnCZUa8J7/w/Dd/96tNjdbfhwY3cKr4CiyehYnN96aWsGyXu66TNxndAxobQvb/k53FwiF7RqobsTAncF5vPFywCIIg2HwbhbSD1jXlnEwgDc1P/LhWFHdLjAZysYCA2UQ4uUiyL8aZxwkfQYgnxC+7LDMIZgVhey4wpvFxbMbAL4aURv0rbFk= ceph-35c0cd22-74d7-11ef-bceb-c7b262605968 2024-09-17T09:30:13.936 DEBUG:teuthology.orchestra.run.smithi136:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC1e+zPEbqBK+IH+X00l5Dwdao98tnmRymQ0tRxmhtBtE536A0prON1FXEWbLv+xxPQQnJuOLMAExLSewt1GUphRjkTATNSoZjkP6p+lr1K57KRONe0mmZAbNZjvIlqlutq4CJWddDPOFREpRT81EHMIUs0y7/wcdJeLcHK6ygqObV67ooXk2IC+zFuyX8c52asrcLx62eW0G7UldySMIresEAwU8IKHsU+pKtVFwmyVri32oj/xhlV53VxoAu9SX3nRvQH5W0T7CdOs/it4JtAHvMvx8vum8bMf0cqkyAsDPsp5PgwMtLElRr8YYwcRAh0Exb3WQuf9PnCZUa8J7/w/Dd/96tNjdbfhwY3cKr4CiyehYnN96aWsGyXu66TNxndAxobQvb/k53FwiF7RqobsTAncF5vPFywCIIg2HwbhbSD1jXlnEwgDc1P/LhWFHdLjAZysYCA2UQ4uUiyL8aZxwkfQYgnxC+7LDMIZgVhey4wpvFxbMbAL4aURv0rbFk= ceph-35c0cd22-74d7-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-17T09:30:14.005 INFO:teuthology.orchestra.run.smithi136.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC1e+zPEbqBK+IH+X00l5Dwdao98tnmRymQ0tRxmhtBtE536A0prON1FXEWbLv+xxPQQnJuOLMAExLSewt1GUphRjkTATNSoZjkP6p+lr1K57KRONe0mmZAbNZjvIlqlutq4CJWddDPOFREpRT81EHMIUs0y7/wcdJeLcHK6ygqObV67ooXk2IC+zFuyX8c52asrcLx62eW0G7UldySMIresEAwU8IKHsU+pKtVFwmyVri32oj/xhlV53VxoAu9SX3nRvQH5W0T7CdOs/it4JtAHvMvx8vum8bMf0cqkyAsDPsp5PgwMtLElRr8YYwcRAh0Exb3WQuf9PnCZUa8J7/w/Dd/96tNjdbfhwY3cKr4CiyehYnN96aWsGyXu66TNxndAxobQvb/k53FwiF7RqobsTAncF5vPFywCIIg2HwbhbSD1jXlnEwgDc1P/LhWFHdLjAZysYCA2UQ4uUiyL8aZxwkfQYgnxC+7LDMIZgVhey4wpvFxbMbAL4aURv0rbFk= ceph-35c0cd22-74d7-11ef-bceb-c7b262605968 2024-09-17T09:30:14.018 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-17T09:30:14.319 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:14 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/2830489041' entity='client.admin' 2024-09-17T09:30:16.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:16 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' 2024-09-17T09:30:16.813 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:16 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd/host:smithi078", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:30:16.813 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:16 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' 2024-09-17T09:30:16.813 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:16 smithi078 ceph-mon[23689]: Deploying daemon alertmanager.smithi078 on smithi078 2024-09-17T09:30:17.568 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-17T09:30:17.568 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-17T09:30:18.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:17 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/929345318' entity='client.admin' 2024-09-17T09:30:20.310 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi136 2024-09-17T09:30:20.310 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2024-09-17T09:30:20.310 DEBUG:teuthology.orchestra.run.smithi136:> dd of=/etc/ceph/ceph.conf 2024-09-17T09:30:20.329 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2024-09-17T09:30:20.330 DEBUG:teuthology.orchestra.run.smithi136:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-17T09:30:20.388 INFO:tasks.cephadm:Adding host smithi136 to orchestrator... 2024-09-17T09:30:20.388 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph orch host add smithi136 2024-09-17T09:30:20.966 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:20 smithi078 ceph-mon[23689]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:30:20.966 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:20 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' 2024-09-17T09:30:23.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:23 smithi078 ceph-mon[23689]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi136", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:30:23.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:23 smithi078 ceph-mon[23689]: Deploying cephadm binary to smithi136 2024-09-17T09:30:23.709 INFO:teuthology.orchestra.run.smithi078.stdout:Added host 'smithi136' with addr '172.21.15.136' 2024-09-17T09:30:24.571 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph orch host ls --format=json 2024-09-17T09:30:24.805 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:24 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' 2024-09-17T09:30:24.805 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:24 smithi078 ceph-mon[23689]: Added host smithi136 2024-09-17T09:30:25.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:25 smithi078 ceph-mon[23689]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:30:27.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:27 smithi078 ceph-mon[23689]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:30:28.743 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-17T09:30:28.744 INFO:teuthology.orchestra.run.smithi078.stdout:[{"addr": "172.21.15.78", "hostname": "smithi078", "labels": [], "status": ""}, {"addr": "172.21.15.136", "hostname": "smithi136", "labels": [], "status": ""}] 2024-09-17T09:30:29.573 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-17T09:30:29.573 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph osd crush tunables default 2024-09-17T09:30:29.709 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:29 smithi078 ceph-mon[23689]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:30:29.709 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:29 smithi078 ceph-mon[23689]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T09:30:31.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:30 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' 2024-09-17T09:30:31.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:30 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T09:30:31.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:30 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-17T09:30:31.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:30 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:30:31.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:30 smithi078 ceph-mon[23689]: Deploying daemon crash.smithi078 on smithi078 2024-09-17T09:30:32.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:31 smithi078 ceph-mon[23689]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:30:32.877 INFO:teuthology.orchestra.run.smithi078.stderr:adjusted tunables profile to default 2024-09-17T09:30:33.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:32 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/579515132' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-17T09:30:33.465 INFO:tasks.cephadm:Adding mon.smithi078 on smithi078 2024-09-17T09:30:33.466 INFO:tasks.cephadm:Adding mon.smithi136 on smithi136 2024-09-17T09:30:33.466 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph orch apply mon '2;smithi078:172.21.15.78=smithi078;smithi136:172.21.15.136=smithi136' 2024-09-17T09:30:34.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:33 smithi078 ceph-mon[23689]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:30:34.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:33 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/579515132' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-17T09:30:34.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:33 smithi078 ceph-mon[23689]: osdmap e4: 0 total, 0 up, 0 in 2024-09-17T09:30:34.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:33 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' 2024-09-17T09:30:34.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:33 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' 2024-09-17T09:30:34.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:33 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' 2024-09-17T09:30:34.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:33 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T09:30:34.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:33 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' 2024-09-17T09:30:35.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:34 smithi078 ceph-mon[23689]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T09:30:35.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:34 smithi078 ceph-mon[23689]: Deploying daemon grafana.smithi078 on smithi078 2024-09-17T09:30:35.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:34 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' 2024-09-17T09:30:35.440 INFO:teuthology.orchestra.run.smithi136.stdout:Scheduled mon update... 2024-09-17T09:30:36.015 DEBUG:teuthology.orchestra.run.smithi136:mon.smithi136> sudo journalctl -f -n 0 -u ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@mon.smithi136.service 2024-09-17T09:30:36.017 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T09:30:36.018 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T09:30:36.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:35 smithi078 ceph-mon[23689]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:30:36.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:35 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' 2024-09-17T09:30:37.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:36 smithi078 ceph-mon[23689]: from='client.14194 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi078:172.21.15.78=smithi078;smithi136:172.21.15.136=smithi136", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:30:37.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:36 smithi078 ceph-mon[23689]: Saving service mon spec with placement smithi078:172.21.15.78=smithi078;smithi136:172.21.15.136=smithi136;count:2 2024-09-17T09:30:37.836 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-17T09:30:37.837 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":1,"fsid":"35c0cd22-74d7-11ef-bceb-c7b262605968","modified":"2024-09-17T09:28:55.619057Z","created":"2024-09-17T09:28:55.619057Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T09:30:37.839 INFO:teuthology.orchestra.run.smithi136.stderr:dumped monmap epoch 1 2024-09-17T09:30:38.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:37 smithi078 ceph-mon[23689]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:30:38.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:37 smithi078 ceph-mon[23689]: from='client.? 172.21.15.136:0/1195035446' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T09:30:39.429 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T09:30:39.430 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T09:30:39.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:39 smithi078 ceph-mon[23689]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:30:41.309 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-17T09:30:41.309 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":1,"fsid":"35c0cd22-74d7-11ef-bceb-c7b262605968","modified":"2024-09-17T09:28:55.619057Z","created":"2024-09-17T09:28:55.619057Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T09:30:41.310 INFO:teuthology.orchestra.run.smithi136.stderr:dumped monmap epoch 1 2024-09-17T09:30:41.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:41 smithi078 ceph-mon[23689]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:30:41.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:41 smithi078 ceph-mon[23689]: from='client.? 172.21.15.136:0/3169325140' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T09:30:42.891 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T09:30:42.891 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T09:30:43.716 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:43 smithi078 ceph-mon[23689]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:30:44.725 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-17T09:30:44.726 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":1,"fsid":"35c0cd22-74d7-11ef-bceb-c7b262605968","modified":"2024-09-17T09:28:55.619057Z","created":"2024-09-17T09:28:55.619057Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T09:30:44.728 INFO:teuthology.orchestra.run.smithi136.stderr:dumped monmap epoch 1 2024-09-17T09:30:45.297 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:44 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' 2024-09-17T09:30:45.297 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:44 smithi078 ceph-mon[23689]: Deploying daemon node-exporter.smithi078 on smithi078 2024-09-17T09:30:45.297 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:44 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' 2024-09-17T09:30:45.297 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:44 smithi078 ceph-mon[23689]: from='client.? 172.21.15.136:0/1451194415' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T09:30:46.303 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T09:30:46.303 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T09:30:46.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:45 smithi078 ceph-mon[23689]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:30:48.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:47 smithi078 ceph-mon[23689]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:30:48.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:47 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' 2024-09-17T09:30:48.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:47 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-17T09:30:48.159 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-17T09:30:48.159 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":1,"fsid":"35c0cd22-74d7-11ef-bceb-c7b262605968","modified":"2024-09-17T09:28:55.619057Z","created":"2024-09-17T09:28:55.619057Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T09:30:48.161 INFO:teuthology.orchestra.run.smithi136.stderr:dumped monmap epoch 1 2024-09-17T09:30:49.034 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:48 smithi078 ceph-mon[23689]: from='mgr.14162 172.21.15.78:0/3161039596' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-17T09:30:49.034 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:48 smithi078 ceph-mon[23689]: mgrmap e14: smithi078.rsqofi(active, since 43s) 2024-09-17T09:30:49.034 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:48 smithi078 ceph-mon[23689]: from='client.? 172.21.15.136:0/22009506' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T09:30:49.781 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T09:30:49.782 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T09:30:51.549 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-17T09:30:51.550 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":1,"fsid":"35c0cd22-74d7-11ef-bceb-c7b262605968","modified":"2024-09-17T09:28:55.619057Z","created":"2024-09-17T09:28:55.619057Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T09:30:51.551 INFO:teuthology.orchestra.run.smithi136.stderr:dumped monmap epoch 1 2024-09-17T09:30:51.921 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:51 smithi078 ceph-mon[23689]: from='client.? 172.21.15.136:0/3080625398' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T09:30:53.062 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T09:30:53.062 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T09:30:54.847 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-17T09:30:54.848 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":1,"fsid":"35c0cd22-74d7-11ef-bceb-c7b262605968","modified":"2024-09-17T09:28:55.619057Z","created":"2024-09-17T09:28:55.619057Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T09:30:54.849 INFO:teuthology.orchestra.run.smithi136.stderr:dumped monmap epoch 1 2024-09-17T09:30:55.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:54 smithi078 ceph-mon[23689]: Active manager daemon smithi078.rsqofi restarted 2024-09-17T09:30:55.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:54 smithi078 ceph-mon[23689]: Activating manager daemon smithi078.rsqofi 2024-09-17T09:30:55.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:54 smithi078 ceph-mon[23689]: osdmap e5: 0 total, 0 up, 0 in 2024-09-17T09:30:55.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:54 smithi078 ceph-mon[23689]: mgrmap e15: smithi078.rsqofi(active, starting, since 0.603885s) 2024-09-17T09:30:55.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:54 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-17T09:30:55.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:54 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mgr metadata", "who": "smithi078.rsqofi", "id": "smithi078.rsqofi"}]: dispatch 2024-09-17T09:30:55.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:54 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T09:30:55.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:54 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T09:30:55.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:54 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T09:30:55.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:54 smithi078 ceph-mon[23689]: Manager daemon smithi078.rsqofi is now available 2024-09-17T09:30:55.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:54 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:30:55.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:54 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:30:55.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:54 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:30:55.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:54 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:30:55.098 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:54 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:30:55.098 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:54 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:30:55.098 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:54 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/trash_purge_schedule"}]: dispatch 2024-09-17T09:30:55.098 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:54 smithi078 ceph-mon[23689]: from='client.? 172.21.15.136:0/1399595930' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T09:30:56.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:56 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:30:56.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:56 smithi078 ceph-mon[23689]: mgrmap e16: smithi078.rsqofi(active, since 1.6062s) 2024-09-17T09:30:56.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:56 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:30:56.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:56 smithi078 ceph-mon[23689]: [17/Sep/2024:09:30:55] ENGINE Bus STARTING 2024-09-17T09:30:56.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:56 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:30:56.604 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T09:30:56.604 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T09:30:57.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:57 smithi078 ceph-mon[23689]: [17/Sep/2024:09:30:55] ENGINE Serving on https://172.21.15.78:7150 2024-09-17T09:30:57.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:57 smithi078 ceph-mon[23689]: [17/Sep/2024:09:30:55] ENGINE Bus STARTED 2024-09-17T09:30:57.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:57 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:30:57.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:57 smithi078 ceph-mon[23689]: mgrmap e17: smithi078.rsqofi(active, since 2s) 2024-09-17T09:30:58.470 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-17T09:30:58.470 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":1,"fsid":"35c0cd22-74d7-11ef-bceb-c7b262605968","modified":"2024-09-17T09:28:55.619057Z","created":"2024-09-17T09:28:55.619057Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T09:30:58.472 INFO:teuthology.orchestra.run.smithi136.stderr:dumped monmap epoch 1 2024-09-17T09:30:59.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:30:59 smithi078 ceph-mon[23689]: from='client.? 172.21.15.136:0/4076865065' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T09:31:01.187 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T09:31:01.187 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T09:31:02.053 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:01 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:02.053 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:01 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:02.053 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:01 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd/host:smithi136", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:31:02.991 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-17T09:31:02.991 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":1,"fsid":"35c0cd22-74d7-11ef-bceb-c7b262605968","modified":"2024-09-17T09:28:55.619057Z","created":"2024-09-17T09:28:55.619057Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T09:31:02.992 INFO:teuthology.orchestra.run.smithi136.stderr:dumped monmap epoch 1 2024-09-17T09:31:03.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:03 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:03.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:03 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:03.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:03 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd/host:smithi078", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:31:03.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:03 smithi078 ceph-mon[23689]: Updating smithi078:/etc/ceph/ceph.conf 2024-09-17T09:31:03.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:03 smithi078 ceph-mon[23689]: from='client.? 172.21.15.136:0/2650550154' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T09:31:03.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:03 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:03.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:03 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:03.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:03 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi136", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T09:31:03.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:03 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi136", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-17T09:31:03.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:03 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:31:04.541 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T09:31:04.541 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T09:31:04.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:04 smithi078 ceph-mon[23689]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-09-17T09:31:04.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:04 smithi078 ceph-mon[23689]: Deploying daemon crash.smithi136 on smithi136 2024-09-17T09:31:06.712 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-17T09:31:06.713 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":1,"fsid":"35c0cd22-74d7-11ef-bceb-c7b262605968","modified":"2024-09-17T09:28:55.619057Z","created":"2024-09-17T09:28:55.619057Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T09:31:06.714 INFO:teuthology.orchestra.run.smithi136.stderr:dumped monmap epoch 1 2024-09-17T09:31:07.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:06 smithi078 ceph-mon[23689]: from='client.? 172.21.15.136:0/4176033465' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T09:31:08.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:07 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:08.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:07 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi136.hicmlm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T09:31:08.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:07 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi136.hicmlm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-17T09:31:08.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:07 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T09:31:08.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:07 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:31:08.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:07 smithi078 ceph-mon[23689]: Deploying daemon mgr.smithi136.hicmlm on smithi136 2024-09-17T09:31:08.551 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T09:31:08.551 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T09:31:10.441 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-17T09:31:10.442 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":1,"fsid":"35c0cd22-74d7-11ef-bceb-c7b262605968","modified":"2024-09-17T09:28:55.619057Z","created":"2024-09-17T09:28:55.619057Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T09:31:10.443 INFO:teuthology.orchestra.run.smithi136.stderr:dumped monmap epoch 1 2024-09-17T09:31:10.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:10 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:10.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:10 smithi078 ceph-mon[23689]: from='client.? 172.21.15.136:0/3652100205' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T09:31:12.348 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:11 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:12.348 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:11 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T09:31:12.348 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:11 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:31:12.349 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:11 smithi078 ceph-mon[23689]: Deploying daemon mon.smithi136 on smithi136 2024-09-17T09:31:12.360 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T09:31:12.360 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T09:31:15.450 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-17T09:31:15.450 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":1,"fsid":"35c0cd22-74d7-11ef-bceb-c7b262605968","modified":"2024-09-17T09:28:55.619057Z","created":"2024-09-17T09:28:55.619057Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T09:31:15.452 INFO:teuthology.orchestra.run.smithi136.stderr:dumped monmap epoch 1 2024-09-17T09:31:17.197 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T09:31:17.197 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T09:31:20.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:20 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-17T09:31:20.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:20 smithi078 ceph-mon[23689]: mon.smithi078 calling monitor election 2024-09-17T09:31:20.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:20 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata", "id": "smithi136"}]: dispatch 2024-09-17T09:31:20.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:20 smithi078 ceph-mon[23689]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:20.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:20 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata", "id": "smithi136"}]: dispatch 2024-09-17T09:31:20.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:20 smithi078 ceph-mon[23689]: from='mgr.? 172.21.15.136:0/3934034944' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi136.hicmlm/crt"}]: dispatch 2024-09-17T09:31:20.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:20 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata", "id": "smithi136"}]: dispatch 2024-09-17T09:31:20.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:20 smithi078 ceph-mon[23689]: mon.smithi136 calling monitor election 2024-09-17T09:31:20.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:20 smithi078 ceph-mon[23689]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:20.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:20 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata", "id": "smithi136"}]: dispatch 2024-09-17T09:31:20.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:20 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata", "id": "smithi136"}]: dispatch 2024-09-17T09:31:20.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:20 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata", "id": "smithi136"}]: dispatch 2024-09-17T09:31:20.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:20 smithi078 ceph-mon[23689]: mon.smithi078 is new leader, mons smithi078,smithi136 in quorum (ranks 0,1) 2024-09-17T09:31:20.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:20 smithi078 ceph-mon[23689]: monmap e2: 2 mons at {smithi078=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0],smithi136=[v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0]} 2024-09-17T09:31:20.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:20 smithi078 ceph-mon[23689]: fsmap 2024-09-17T09:31:20.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:20 smithi078 ceph-mon[23689]: osdmap e5: 0 total, 0 up, 0 in 2024-09-17T09:31:20.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:20 smithi078 ceph-mon[23689]: mgrmap e17: smithi078.rsqofi(active, since 26s) 2024-09-17T09:31:20.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:20 smithi078 ceph-mon[23689]: Standby manager daemon smithi136.hicmlm started 2024-09-17T09:31:20.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:20 smithi078 ceph-mon[23689]: from='mgr.? 172.21.15.136:0/3934034944' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T09:31:20.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:20 smithi078 ceph-mon[23689]: from='mgr.? 172.21.15.136:0/3934034944' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi136.hicmlm/key"}]: dispatch 2024-09-17T09:31:20.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:20 smithi078 ceph-mon[23689]: from='mgr.? 172.21.15.136:0/3934034944' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T09:31:20.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:20 smithi078 ceph-mon[23689]: overall HEALTH_OK 2024-09-17T09:31:20.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:20 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:21.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:21 smithi078 ceph-mon[23689]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:21.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:21 smithi078 ceph-mon[23689]: Deploying daemon node-exporter.smithi136 on smithi136 2024-09-17T09:31:21.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:21 smithi078 ceph-mon[23689]: mgrmap e18: smithi078.rsqofi(active, since 26s), standbys: smithi136.hicmlm 2024-09-17T09:31:21.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:21 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mgr metadata", "who": "smithi136.hicmlm", "id": "smithi136.hicmlm"}]: dispatch 2024-09-17T09:31:21.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:21 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata", "id": "smithi136"}]: dispatch 2024-09-17T09:31:22.006 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-17T09:31:22.006 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":2,"fsid":"35c0cd22-74d7-11ef-bceb-c7b262605968","modified":"2024-09-17T09:31:15.473949Z","created":"2024-09-17T09:28:55.619057Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi136","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:3300","nonce":0},{"type":"v1","addr":"172.21.15.136:6789","nonce":0}]},"addr":"172.21.15.136:6789/0","public_addr":"172.21.15.136:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-17T09:31:22.006 INFO:teuthology.orchestra.run.smithi136.stderr:dumped monmap epoch 2 2024-09-17T09:31:22.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:22 smithi078 ceph-mon[23689]: from='client.? 172.21.15.136:0/1599572308' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T09:31:22.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:22 smithi136 ceph-mon[25953]: from='client.? 172.21.15.136:0/1599572308' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T09:31:23.725 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-17T09:31:23.725 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph config generate-minimal-conf 2024-09-17T09:31:23.794 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:23 smithi136 ceph-mon[25953]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:23.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:23 smithi078 ceph-mon[23689]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:25.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:25 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:25.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:25 smithi136 ceph-mon[25953]: Deploying daemon prometheus.smithi078 on smithi078 2024-09-17T09:31:25.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:25 smithi136 ceph-mon[25953]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:25.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:25 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:25.543 INFO:teuthology.orchestra.run.smithi078.stdout:# minimal ceph.conf for 35c0cd22-74d7-11ef-bceb-c7b262605968 2024-09-17T09:31:25.544 INFO:teuthology.orchestra.run.smithi078.stdout:[global] 2024-09-17T09:31:25.544 INFO:teuthology.orchestra.run.smithi078.stdout: fsid = 35c0cd22-74d7-11ef-bceb-c7b262605968 2024-09-17T09:31:25.544 INFO:teuthology.orchestra.run.smithi078.stdout: mon_host = [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] [v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0] 2024-09-17T09:31:25.544 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:25 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:25.544 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:25 smithi078 ceph-mon[23689]: Deploying daemon prometheus.smithi078 on smithi078 2024-09-17T09:31:25.544 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:25 smithi078 ceph-mon[23689]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:25.544 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:25 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:26.149 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-17T09:31:26.149 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-17T09:31:26.149 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/ceph/ceph.conf 2024-09-17T09:31:26.177 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-17T09:31:26.177 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-17T09:31:26.243 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2024-09-17T09:31:26.243 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd of=/etc/ceph/ceph.conf 2024-09-17T09:31:26.276 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2024-09-17T09:31:26.276 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-17T09:31:26.344 DEBUG:tasks.cephadm:set 0 configs 2024-09-17T09:31:26.345 INFO:tasks.cephadm:Deploying OSDs... 2024-09-17T09:31:26.345 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-17T09:31:26.345 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/scratch_devs of=/dev/stdout 2024-09-17T09:31:26.361 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-17T09:31:26.361 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_1 2024-09-17T09:31:26.417 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-17T09:31:26.417 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T09:31:26.417 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-09-17T09:31:26.418 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T09:31:26.418 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T09:31:26.418 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-09-17 09:30:44.597867607 +0000 2024-09-17T09:31:26.418 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-09-17 09:28:14.738490062 +0000 2024-09-17T09:31:26.418 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-09-17 09:28:14.738490062 +0000 2024-09-17T09:31:26.418 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-09-17 09:28:14.738490062 +0000 2024-09-17T09:31:26.419 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-17T09:31:26.480 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-09-17T09:31:26.480 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-09-17T09:31:26.481 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000270141 s, 1.9 MB/s 2024-09-17T09:31:26.481 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-17T09:31:26.538 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_2 2024-09-17T09:31:26.593 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-17T09:31:26.594 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T09:31:26.594 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-09-17T09:31:26.594 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T09:31:26.594 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T09:31:26.594 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-09-17 09:30:44.597867607 +0000 2024-09-17T09:31:26.594 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-09-17 09:28:15.076490690 +0000 2024-09-17T09:31:26.594 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-09-17 09:28:15.076490690 +0000 2024-09-17T09:31:26.594 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-09-17 09:28:15.076490690 +0000 2024-09-17T09:31:26.594 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-17T09:31:26.657 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-09-17T09:31:26.658 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-09-17T09:31:26.658 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000264119 s, 1.9 MB/s 2024-09-17T09:31:26.659 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-17T09:31:26.715 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_3 2024-09-17T09:31:26.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:26 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/2791790730' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:31:26.770 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-17T09:31:26.771 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T09:31:26.771 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-09-17T09:31:26.771 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T09:31:26.771 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T09:31:26.771 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-09-17 09:30:44.598867608 +0000 2024-09-17T09:31:26.771 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-09-17 09:28:15.400491293 +0000 2024-09-17T09:31:26.771 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-09-17 09:28:15.400491293 +0000 2024-09-17T09:31:26.771 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-09-17 09:28:15.400491293 +0000 2024-09-17T09:31:26.771 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-17T09:31:26.832 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:26 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/2791790730' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:31:26.834 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-09-17T09:31:26.834 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-09-17T09:31:26.834 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000189735 s, 2.7 MB/s 2024-09-17T09:31:26.836 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-17T09:31:26.892 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_4 2024-09-17T09:31:26.947 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-17T09:31:26.947 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T09:31:26.947 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-09-17T09:31:26.948 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T09:31:26.948 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T09:31:26.948 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-09-17 09:30:44.598867608 +0000 2024-09-17T09:31:26.948 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-09-17 09:28:15.742491929 +0000 2024-09-17T09:31:26.948 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-09-17 09:28:15.742491929 +0000 2024-09-17T09:31:26.948 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-09-17 09:28:15.742491929 +0000 2024-09-17T09:31:26.948 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-17T09:31:27.014 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-09-17T09:31:27.015 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-09-17T09:31:27.015 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000196168 s, 2.6 MB/s 2024-09-17T09:31:27.016 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-17T09:31:27.074 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2024-09-17T09:31:27.074 DEBUG:teuthology.orchestra.run.smithi136:> dd if=/scratch_devs of=/dev/stdout 2024-09-17T09:31:27.093 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-17T09:31:27.093 DEBUG:teuthology.orchestra.run.smithi136:> stat /dev/vg_nvme/lv_1 2024-09-17T09:31:27.151 INFO:teuthology.orchestra.run.smithi136.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-17T09:31:27.152 INFO:teuthology.orchestra.run.smithi136.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T09:31:27.152 INFO:teuthology.orchestra.run.smithi136.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-09-17T09:31:27.152 INFO:teuthology.orchestra.run.smithi136.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T09:31:27.152 INFO:teuthology.orchestra.run.smithi136.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T09:31:27.152 INFO:teuthology.orchestra.run.smithi136.stdout:Access: 2024-09-17 09:31:21.077413592 +0000 2024-09-17T09:31:27.152 INFO:teuthology.orchestra.run.smithi136.stdout:Modify: 2024-09-17 09:28:07.989072601 +0000 2024-09-17T09:31:27.152 INFO:teuthology.orchestra.run.smithi136.stdout:Change: 2024-09-17 09:28:07.989072601 +0000 2024-09-17T09:31:27.152 INFO:teuthology.orchestra.run.smithi136.stdout: Birth: 2024-09-17 09:28:07.989072601 +0000 2024-09-17T09:31:27.152 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-17T09:31:27.219 INFO:teuthology.orchestra.run.smithi136.stderr:1+0 records in 2024-09-17T09:31:27.220 INFO:teuthology.orchestra.run.smithi136.stderr:1+0 records out 2024-09-17T09:31:27.220 INFO:teuthology.orchestra.run.smithi136.stderr:512 bytes copied, 0.000242119 s, 2.1 MB/s 2024-09-17T09:31:27.221 DEBUG:teuthology.orchestra.run.smithi136:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-17T09:31:27.280 DEBUG:teuthology.orchestra.run.smithi136:> stat /dev/vg_nvme/lv_2 2024-09-17T09:31:27.338 INFO:teuthology.orchestra.run.smithi136.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-17T09:31:27.338 INFO:teuthology.orchestra.run.smithi136.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T09:31:27.338 INFO:teuthology.orchestra.run.smithi136.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-09-17T09:31:27.338 INFO:teuthology.orchestra.run.smithi136.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T09:31:27.338 INFO:teuthology.orchestra.run.smithi136.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T09:31:27.338 INFO:teuthology.orchestra.run.smithi136.stdout:Access: 2024-09-17 09:31:21.078413570 +0000 2024-09-17T09:31:27.338 INFO:teuthology.orchestra.run.smithi136.stdout:Modify: 2024-09-17 09:28:08.300065539 +0000 2024-09-17T09:31:27.338 INFO:teuthology.orchestra.run.smithi136.stdout:Change: 2024-09-17 09:28:08.300065539 +0000 2024-09-17T09:31:27.338 INFO:teuthology.orchestra.run.smithi136.stdout: Birth: 2024-09-17 09:28:08.300065539 +0000 2024-09-17T09:31:27.339 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-17T09:31:27.404 INFO:teuthology.orchestra.run.smithi136.stderr:1+0 records in 2024-09-17T09:31:27.404 INFO:teuthology.orchestra.run.smithi136.stderr:1+0 records out 2024-09-17T09:31:27.404 INFO:teuthology.orchestra.run.smithi136.stderr:512 bytes copied, 0.000218406 s, 2.3 MB/s 2024-09-17T09:31:27.405 DEBUG:teuthology.orchestra.run.smithi136:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-17T09:31:27.464 DEBUG:teuthology.orchestra.run.smithi136:> stat /dev/vg_nvme/lv_3 2024-09-17T09:31:27.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:27 smithi136 ceph-mon[25953]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:27.491 INFO:teuthology.orchestra.run.smithi136.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-17T09:31:27.491 INFO:teuthology.orchestra.run.smithi136.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T09:31:27.491 INFO:teuthology.orchestra.run.smithi136.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-09-17T09:31:27.491 INFO:teuthology.orchestra.run.smithi136.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T09:31:27.491 INFO:teuthology.orchestra.run.smithi136.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T09:31:27.491 INFO:teuthology.orchestra.run.smithi136.stdout:Access: 2024-09-17 09:31:21.078413570 +0000 2024-09-17T09:31:27.492 INFO:teuthology.orchestra.run.smithi136.stdout:Modify: 2024-09-17 09:28:08.643057750 +0000 2024-09-17T09:31:27.492 INFO:teuthology.orchestra.run.smithi136.stdout:Change: 2024-09-17 09:28:08.643057750 +0000 2024-09-17T09:31:27.492 INFO:teuthology.orchestra.run.smithi136.stdout: Birth: 2024-09-17 09:28:08.643057750 +0000 2024-09-17T09:31:27.492 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-17T09:31:27.557 INFO:teuthology.orchestra.run.smithi136.stderr:1+0 records in 2024-09-17T09:31:27.557 INFO:teuthology.orchestra.run.smithi136.stderr:1+0 records out 2024-09-17T09:31:27.557 INFO:teuthology.orchestra.run.smithi136.stderr:512 bytes copied, 0.000235004 s, 2.2 MB/s 2024-09-17T09:31:27.558 DEBUG:teuthology.orchestra.run.smithi136:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-17T09:31:27.616 DEBUG:teuthology.orchestra.run.smithi136:> stat /dev/vg_nvme/lv_4 2024-09-17T09:31:27.673 INFO:teuthology.orchestra.run.smithi136.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-17T09:31:27.673 INFO:teuthology.orchestra.run.smithi136.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T09:31:27.673 INFO:teuthology.orchestra.run.smithi136.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-09-17T09:31:27.673 INFO:teuthology.orchestra.run.smithi136.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T09:31:27.673 INFO:teuthology.orchestra.run.smithi136.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T09:31:27.673 INFO:teuthology.orchestra.run.smithi136.stdout:Access: 2024-09-17 09:31:21.078413570 +0000 2024-09-17T09:31:27.673 INFO:teuthology.orchestra.run.smithi136.stdout:Modify: 2024-09-17 09:28:08.965050439 +0000 2024-09-17T09:31:27.673 INFO:teuthology.orchestra.run.smithi136.stdout:Change: 2024-09-17 09:28:08.965050439 +0000 2024-09-17T09:31:27.673 INFO:teuthology.orchestra.run.smithi136.stdout: Birth: 2024-09-17 09:28:08.965050439 +0000 2024-09-17T09:31:27.674 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-17T09:31:27.738 INFO:teuthology.orchestra.run.smithi136.stderr:1+0 records in 2024-09-17T09:31:27.738 INFO:teuthology.orchestra.run.smithi136.stderr:1+0 records out 2024-09-17T09:31:27.738 INFO:teuthology.orchestra.run.smithi136.stderr:512 bytes copied, 0.000231652 s, 2.2 MB/s 2024-09-17T09:31:27.739 DEBUG:teuthology.orchestra.run.smithi136:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-17T09:31:27.796 INFO:tasks.cephadm:Deploying osd.0 on smithi078 with /dev/vg_nvme/lv_4... 2024-09-17T09:31:27.796 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-17T09:31:27.818 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:27 smithi078 ceph-mon[23689]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:29.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:29 smithi136 ceph-mon[25953]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:29.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:29 smithi078 ceph-mon[23689]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:31.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:31 smithi136 ceph-mon[25953]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:31.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:31 smithi078 ceph-mon[23689]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:33.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:33 smithi078 ceph-mon[23689]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:33.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:33 smithi136 ceph-mon[25953]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:35.798 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:35 smithi078 ceph-mon[23689]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:35.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:35 smithi136 ceph-mon[25953]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:37.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:36 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:37.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:36 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:31:37.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:36 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:31:37.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:36 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:31:37.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:36 smithi078 ceph-mon[23689]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:37.235 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:36 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:37.236 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:36 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:31:37.236 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:36 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:31:37.236 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:36 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:31:37.236 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:36 smithi136 ceph-mon[25953]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:37.725 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-17T09:31:37.743 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi078:vg_nvme/lv_4 2024-09-17T09:31:38.960 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:38 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:38.961 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:38 smithi078 ceph-mon[23689]: Updating smithi136:/etc/ceph/ceph.conf 2024-09-17T09:31:38.961 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:38 smithi078 ceph-mon[23689]: Updating smithi136:/etc/ceph/ceph.client.admin.keyring 2024-09-17T09:31:38.961 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:38 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:38.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:38 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:38.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:38 smithi136 ceph-mon[25953]: Updating smithi136:/etc/ceph/ceph.conf 2024-09-17T09:31:38.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:38 smithi136 ceph-mon[25953]: Updating smithi136:/etc/ceph/ceph.client.admin.keyring 2024-09-17T09:31:38.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:38 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:39.968 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:39 smithi078 ceph-mon[23689]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:39.969 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:39 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:39.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:39 smithi136 ceph-mon[25953]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:39.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:39 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:40.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:40 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T09:31:40.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:40 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T09:31:40.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:40 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:31:40.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:40 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T09:31:40.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:40 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T09:31:40.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:40 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:31:41.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:41 smithi136 ceph-mon[25953]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:41.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:41 smithi136 ceph-mon[25953]: from='client.14238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:31:42.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:41 smithi078 ceph-mon[23689]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:42.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:41 smithi078 ceph-mon[23689]: from='client.14238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:31:43.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:43 smithi136 ceph-mon[25953]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:43.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:43 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:43.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:43 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:43.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:43 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:43.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:43 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T09:31:43.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:43 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T09:31:43.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:43 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:31:44.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:43 smithi078 ceph-mon[23689]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:44.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:43 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:44.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:43 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:44.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:43 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:44.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:43 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T09:31:44.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:43 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T09:31:44.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:43 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:31:44.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:44 smithi136 ceph-mon[25953]: Updating smithi078:/etc/ceph/ceph.conf 2024-09-17T09:31:44.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:44 smithi136 ceph-mon[25953]: Reconfiguring mon.smithi078 (unknown last config time)... 2024-09-17T09:31:44.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:44 smithi136 ceph-mon[25953]: Reconfiguring daemon mon.smithi078 on smithi078 2024-09-17T09:31:45.022 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:44 smithi078 ceph-mon[23689]: Updating smithi078:/etc/ceph/ceph.conf 2024-09-17T09:31:45.022 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:44 smithi078 ceph-mon[23689]: Reconfiguring mon.smithi078 (unknown last config time)... 2024-09-17T09:31:45.022 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:44 smithi078 ceph-mon[23689]: Reconfiguring daemon mon.smithi078 on smithi078 2024-09-17T09:31:45.750 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:45 smithi078 ceph-mon[23689]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:45.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:45 smithi136 ceph-mon[25953]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:47.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:46 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:47.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:46 smithi078 ceph-mon[23689]: Reconfiguring mgr.smithi078.rsqofi (unknown last config time)... 2024-09-17T09:31:47.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:46 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.rsqofi", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T09:31:47.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:46 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T09:31:47.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:46 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:31:47.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:46 smithi078 ceph-mon[23689]: Reconfiguring daemon mgr.smithi078.rsqofi on smithi078 2024-09-17T09:31:47.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:46 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/4229019286' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6666cf1b-b4e3-48d3-a076-be328ec75fb6"}]: dispatch 2024-09-17T09:31:47.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:46 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/4229019286' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6666cf1b-b4e3-48d3-a076-be328ec75fb6"}]': finished 2024-09-17T09:31:47.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:46 smithi078 ceph-mon[23689]: osdmap e6: 1 total, 0 up, 1 in 2024-09-17T09:31:47.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:46 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:31:47.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:46 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:47.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:46 smithi136 ceph-mon[25953]: Reconfiguring mgr.smithi078.rsqofi (unknown last config time)... 2024-09-17T09:31:47.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:46 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.rsqofi", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T09:31:47.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:46 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T09:31:47.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:46 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:31:47.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:46 smithi136 ceph-mon[25953]: Reconfiguring daemon mgr.smithi078.rsqofi on smithi078 2024-09-17T09:31:47.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:46 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/4229019286' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6666cf1b-b4e3-48d3-a076-be328ec75fb6"}]: dispatch 2024-09-17T09:31:47.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:46 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/4229019286' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6666cf1b-b4e3-48d3-a076-be328ec75fb6"}]': finished 2024-09-17T09:31:47.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:46 smithi136 ceph-mon[25953]: osdmap e6: 1 total, 0 up, 1 in 2024-09-17T09:31:47.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:46 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:31:48.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:47 smithi078 ceph-mon[23689]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:48.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:47 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/2784400711' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T09:31:48.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:47 smithi136 ceph-mon[25953]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:48.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:47 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/2784400711' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T09:31:49.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:49 smithi136 ceph-mon[25953]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:49.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:49 smithi078 ceph-mon[23689]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:51.728 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:51 smithi136 ceph-mon[25953]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:51.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:51 smithi078 ceph-mon[23689]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:53.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:53 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:53.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:53 smithi136 ceph-mon[25953]: Reconfiguring alertmanager.smithi078 (dependencies changed)... 2024-09-17T09:31:53.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:53 smithi136 ceph-mon[25953]: Reconfiguring daemon alertmanager.smithi078 on smithi078 2024-09-17T09:31:53.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:53 smithi136 ceph-mon[25953]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:53.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:53 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:53.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:53 smithi078 ceph-mon[23689]: Reconfiguring alertmanager.smithi078 (dependencies changed)... 2024-09-17T09:31:53.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:53 smithi078 ceph-mon[23689]: Reconfiguring daemon alertmanager.smithi078 on smithi078 2024-09-17T09:31:53.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:53 smithi078 ceph-mon[23689]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:54.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:54 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T09:31:54.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:54 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:31:54.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:54 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T09:31:54.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:54 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:31:55.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:55 smithi136 ceph-mon[25953]: Deploying daemon osd.0 on smithi078 2024-09-17T09:31:55.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:55 smithi136 ceph-mon[25953]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:55.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:55 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:31:55.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:55 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/trash_purge_schedule"}]: dispatch 2024-09-17T09:31:55.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:55 smithi078 ceph-mon[23689]: Deploying daemon osd.0 on smithi078 2024-09-17T09:31:55.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:55 smithi078 ceph-mon[23689]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:55.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:55 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:31:55.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:55 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/trash_purge_schedule"}]: dispatch 2024-09-17T09:31:57.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:57 smithi136 ceph-mon[25953]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:57.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:57 smithi078 ceph-mon[23689]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:59.564 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:59 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:59.564 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:59 smithi078 ceph-mon[23689]: Reconfiguring crash.smithi078 (monmap changed)... 2024-09-17T09:31:59.564 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:59 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T09:31:59.564 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:59 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:31:59.564 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:59 smithi078 ceph-mon[23689]: Reconfiguring daemon crash.smithi078 on smithi078 2024-09-17T09:31:59.565 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:31:59 smithi078 ceph-mon[23689]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:31:59.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:59 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:31:59.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:59 smithi136 ceph-mon[25953]: Reconfiguring crash.smithi078 (monmap changed)... 2024-09-17T09:31:59.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:59 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T09:31:59.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:59 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:31:59.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:59 smithi136 ceph-mon[25953]: Reconfiguring daemon crash.smithi078 on smithi078 2024-09-17T09:31:59.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:31:59 smithi136 ceph-mon[25953]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:32:01.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:01 smithi136 ceph-mon[25953]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:32:01.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:01 smithi078 ceph-mon[23689]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:32:03.735 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:03 smithi078 ceph-mon[23689]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:32:03.736 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:03 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:03.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:03 smithi136 ceph-mon[25953]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:32:03.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:03 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:05.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:05 smithi078 ceph-mon[23689]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:32:05.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:05 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:05.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:05 smithi078 ceph-mon[23689]: Reconfiguring grafana.smithi078 (dependencies changed)... 2024-09-17T09:32:05.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:05 smithi078 ceph-mon[23689]: Reconfiguring daemon grafana.smithi078 on smithi078 2024-09-17T09:32:05.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:05 smithi136 ceph-mon[25953]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:32:05.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:05 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:05.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:05 smithi136 ceph-mon[25953]: Reconfiguring grafana.smithi078 (dependencies changed)... 2024-09-17T09:32:05.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:05 smithi136 ceph-mon[25953]: Reconfiguring daemon grafana.smithi078 on smithi078 2024-09-17T09:32:06.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:06 smithi078 ceph-mon[23689]: from='osd.0 [v2:172.21.15.78:6802/2296669224,v1:172.21.15.78:6803/2296669224]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-17T09:32:06.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:06 smithi136 ceph-mon[25953]: from='osd.0 [v2:172.21.15.78:6802/2296669224,v1:172.21.15.78:6803/2296669224]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-17T09:32:07.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:07 smithi078 ceph-mon[23689]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:32:07.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:07 smithi078 ceph-mon[23689]: from='osd.0 [v2:172.21.15.78:6802/2296669224,v1:172.21.15.78:6803/2296669224]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-17T09:32:07.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:07 smithi078 ceph-mon[23689]: osdmap e7: 1 total, 0 up, 1 in 2024-09-17T09:32:07.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:07 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:32:07.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:07 smithi078 ceph-mon[23689]: from='osd.0 [v2:172.21.15.78:6802/2296669224,v1:172.21.15.78:6803/2296669224]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-09-17T09:32:07.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:07 smithi136 ceph-mon[25953]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:32:07.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:07 smithi136 ceph-mon[25953]: from='osd.0 [v2:172.21.15.78:6802/2296669224,v1:172.21.15.78:6803/2296669224]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-17T09:32:07.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:07 smithi136 ceph-mon[25953]: osdmap e7: 1 total, 0 up, 1 in 2024-09-17T09:32:07.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:07 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:32:07.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:07 smithi136 ceph-mon[25953]: from='osd.0 [v2:172.21.15.78:6802/2296669224,v1:172.21.15.78:6803/2296669224]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-09-17T09:32:08.805 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:08 smithi078 ceph-mon[23689]: from='osd.0 [v2:172.21.15.78:6802/2296669224,v1:172.21.15.78:6803/2296669224]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-09-17T09:32:08.805 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:08 smithi078 ceph-mon[23689]: osdmap e8: 1 total, 0 up, 1 in 2024-09-17T09:32:08.805 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:08 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:32:08.805 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:08 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:32:08.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:08 smithi136 ceph-mon[25953]: from='osd.0 [v2:172.21.15.78:6802/2296669224,v1:172.21.15.78:6803/2296669224]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-09-17T09:32:08.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:08 smithi136 ceph-mon[25953]: osdmap e8: 1 total, 0 up, 1 in 2024-09-17T09:32:08.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:08 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:32:08.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:08 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:32:09.687 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:09 smithi078 ceph-mon[23689]: purged_snaps scrub starts 2024-09-17T09:32:09.687 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:09 smithi078 ceph-mon[23689]: purged_snaps scrub ok 2024-09-17T09:32:09.687 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:09 smithi078 ceph-mon[23689]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:32:09.687 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:09 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:32:09.687 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:09 smithi078 ceph-mon[23689]: osd.0 [v2:172.21.15.78:6802/2296669224,v1:172.21.15.78:6803/2296669224] boot 2024-09-17T09:32:09.687 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:09 smithi078 ceph-mon[23689]: osdmap e9: 1 total, 1 up, 1 in 2024-09-17T09:32:09.687 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:09 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:32:09.688 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:09 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:09.688 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:09 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi136", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T09:32:09.688 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:09 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:32:09.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:09 smithi136 ceph-mon[25953]: purged_snaps scrub starts 2024-09-17T09:32:09.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:09 smithi136 ceph-mon[25953]: purged_snaps scrub ok 2024-09-17T09:32:09.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:09 smithi136 ceph-mon[25953]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T09:32:09.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:09 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:32:09.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:09 smithi136 ceph-mon[25953]: osd.0 [v2:172.21.15.78:6802/2296669224,v1:172.21.15.78:6803/2296669224] boot 2024-09-17T09:32:09.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:09 smithi136 ceph-mon[25953]: osdmap e9: 1 total, 1 up, 1 in 2024-09-17T09:32:09.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:09 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:32:09.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:09 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:09.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:09 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi136", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T09:32:09.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:09 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:32:10.824 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:10 smithi078 ceph-mon[23689]: Reconfiguring crash.smithi136 (monmap changed)... 2024-09-17T09:32:10.825 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:10 smithi078 ceph-mon[23689]: Reconfiguring daemon crash.smithi136 on smithi136 2024-09-17T09:32:10.825 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:10 smithi078 ceph-mon[23689]: osdmap e10: 1 total, 1 up, 1 in 2024-09-17T09:32:10.828 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:10 smithi136 ceph-mon[25953]: Reconfiguring crash.smithi136 (monmap changed)... 2024-09-17T09:32:10.828 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:10 smithi136 ceph-mon[25953]: Reconfiguring daemon crash.smithi136 on smithi136 2024-09-17T09:32:10.828 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:10 smithi136 ceph-mon[25953]: osdmap e10: 1 total, 1 up, 1 in 2024-09-17T09:32:10.857 INFO:teuthology.orchestra.run.smithi078.stdout:Created osd(s) 0 on host 'smithi078' 2024-09-17T09:32:11.513 DEBUG:teuthology.orchestra.run.smithi078:osd.0> sudo journalctl -f -n 0 -u ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.0.service 2024-09-17T09:32:11.515 INFO:tasks.cephadm:Deploying osd.1 on smithi078 with /dev/vg_nvme/lv_3... 2024-09-17T09:32:11.516 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-17T09:32:12.039 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:11 smithi136 ceph-mon[25953]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:12.039 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:11 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:12.039 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:11 smithi136 ceph-mon[25953]: Reconfiguring mgr.smithi136.hicmlm (monmap changed)... 2024-09-17T09:32:12.039 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:11 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi136.hicmlm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T09:32:12.039 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:11 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T09:32:12.039 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:11 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:32:12.039 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:11 smithi136 ceph-mon[25953]: Reconfiguring daemon mgr.smithi136.hicmlm on smithi136 2024-09-17T09:32:12.039 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:11 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:12.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:11 smithi078 ceph-mon[23689]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:12.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:11 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:12.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:11 smithi078 ceph-mon[23689]: Reconfiguring mgr.smithi136.hicmlm (monmap changed)... 2024-09-17T09:32:12.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:11 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi136.hicmlm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T09:32:12.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:11 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T09:32:12.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:11 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:32:12.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:11 smithi078 ceph-mon[23689]: Reconfiguring daemon mgr.smithi136.hicmlm on smithi136 2024-09-17T09:32:12.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:11 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:13.258 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:13 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:13.258 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:13 smithi136 ceph-mon[25953]: Reconfiguring mon.smithi136 (monmap changed)... 2024-09-17T09:32:13.258 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:13 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T09:32:13.258 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:13 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T09:32:13.258 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:13 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:32:13.258 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:13 smithi136 ceph-mon[25953]: Reconfiguring daemon mon.smithi136 on smithi136 2024-09-17T09:32:13.258 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:13 smithi136 ceph-mon[25953]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:13.336 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:13 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:13.336 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:13 smithi078 ceph-mon[23689]: Reconfiguring mon.smithi136 (monmap changed)... 2024-09-17T09:32:13.336 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:13 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T09:32:13.336 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:13 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T09:32:13.336 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:13 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:32:13.336 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:13 smithi078 ceph-mon[23689]: Reconfiguring daemon mon.smithi136 on smithi136 2024-09-17T09:32:13.336 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:13 smithi078 ceph-mon[23689]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:14.181 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-17T09:32:14.202 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi078:vg_nvme/lv_3 2024-09-17T09:32:14.429 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:14 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:14.429 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:14 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T09:32:14.429 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:14 smithi078 ceph-mon[23689]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T09:32:14.429 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:14 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.78:9093"}]: dispatch 2024-09-17T09:32:14.429 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:14 smithi078 ceph-mon[23689]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.78:9093"}]: dispatch 2024-09-17T09:32:14.430 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:14 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:14.430 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:14 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T09:32:14.430 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:14 smithi078 ceph-mon[23689]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T09:32:14.430 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:14 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.78:3000"}]: dispatch 2024-09-17T09:32:14.430 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:14 smithi078 ceph-mon[23689]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.78:3000"}]: dispatch 2024-09-17T09:32:14.430 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:14 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:14.430 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:14 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:32:14.430 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:14 smithi078 ceph-mon[23689]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:32:14.430 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:14 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.78:9095"}]: dispatch 2024-09-17T09:32:14.430 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:14 smithi078 ceph-mon[23689]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.78:9095"}]: dispatch 2024-09-17T09:32:14.430 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:14 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:14.430 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:14 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:32:14.430 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:14 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:32:14.430 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:14 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:32:14.488 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:14 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:14.488 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:14 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T09:32:14.488 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:14 smithi136 ceph-mon[25953]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T09:32:14.488 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:14 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.78:9093"}]: dispatch 2024-09-17T09:32:14.488 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:14 smithi136 ceph-mon[25953]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.78:9093"}]: dispatch 2024-09-17T09:32:14.488 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:14 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:14.488 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:14 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T09:32:14.488 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:14 smithi136 ceph-mon[25953]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T09:32:14.488 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:14 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.78:3000"}]: dispatch 2024-09-17T09:32:14.488 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:14 smithi136 ceph-mon[25953]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.78:3000"}]: dispatch 2024-09-17T09:32:14.488 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:14 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:14.489 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:14 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:32:14.489 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:14 smithi136 ceph-mon[25953]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:32:14.489 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:14 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.78:9095"}]: dispatch 2024-09-17T09:32:14.489 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:14 smithi136 ceph-mon[25953]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.78:9095"}]: dispatch 2024-09-17T09:32:14.489 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:14 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:14.489 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:14 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:32:14.489 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:14 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:32:14.489 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:14 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:32:15.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:15 smithi078 ceph-mon[23689]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:15.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:15 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:15.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:15 smithi136 ceph-mon[25953]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:15.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:15 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:17.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:17 smithi078 ceph-mon[23689]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:17.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:17 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T09:32:17.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:17 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T09:32:17.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:17 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:32:17.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:17 smithi136 ceph-mon[25953]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:17.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:17 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T09:32:17.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:17 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T09:32:17.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:17 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:32:18.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:18 smithi078 ceph-mon[23689]: from='client.14256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:32:18.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:18 smithi136 ceph-mon[25953]: from='client.14256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:32:19.816 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:19 smithi078 ceph-mon[23689]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:19.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:19 smithi136 ceph-mon[25953]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:21.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:21 smithi078 ceph-mon[23689]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:21.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:21 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:21.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:21 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/2272012005' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5d895d59-82d7-40da-9591-42d8f9b258b3"}]: dispatch 2024-09-17T09:32:21.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:21 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/2272012005' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5d895d59-82d7-40da-9591-42d8f9b258b3"}]': finished 2024-09-17T09:32:21.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:21 smithi078 ceph-mon[23689]: osdmap e11: 2 total, 1 up, 2 in 2024-09-17T09:32:21.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:21 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:32:21.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:21 smithi136 ceph-mon[25953]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:21.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:21 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:21.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:21 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/2272012005' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5d895d59-82d7-40da-9591-42d8f9b258b3"}]: dispatch 2024-09-17T09:32:21.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:21 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/2272012005' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5d895d59-82d7-40da-9591-42d8f9b258b3"}]': finished 2024-09-17T09:32:21.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:21 smithi136 ceph-mon[25953]: osdmap e11: 2 total, 1 up, 2 in 2024-09-17T09:32:21.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:21 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:32:22.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:22 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/500974089' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T09:32:22.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:22 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/500974089' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T09:32:23.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:23 smithi078 ceph-mon[23689]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:23.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:23 smithi136 ceph-mon[25953]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:25.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:25 smithi078 ceph-mon[23689]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:25.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:25 smithi136 ceph-mon[25953]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:27.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:27 smithi078 ceph-mon[23689]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:27.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:27 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:27.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:27 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:32:27.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:27 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:27.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:27 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:27.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:27 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:32:27.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:27 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:32:27.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:27 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:32:27.868 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:27 smithi136 ceph-mon[25953]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:27.868 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:27 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:27.868 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:27 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:32:27.868 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:27 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:27.868 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:27 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:27.868 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:27 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:32:27.868 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:27 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:32:27.868 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:27 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:32:28.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:28 smithi078 ceph-mon[23689]: Detected new or changed devices on smithi078 2024-09-17T09:32:28.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:28 smithi078 ceph-mon[23689]: Adjusting osd_memory_target on smithi078 to 12864M 2024-09-17T09:32:28.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:28 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T09:32:28.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:28 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:32:28.915 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:28 smithi136 ceph-mon[25953]: Detected new or changed devices on smithi078 2024-09-17T09:32:28.915 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:28 smithi136 ceph-mon[25953]: Adjusting osd_memory_target on smithi078 to 12864M 2024-09-17T09:32:28.915 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:28 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T09:32:28.915 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:28 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:32:29.829 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:29 smithi078 ceph-mon[23689]: Deploying daemon osd.1 on smithi078 2024-09-17T09:32:29.829 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:29 smithi078 ceph-mon[23689]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:29.829 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:29 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:29.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:29 smithi136 ceph-mon[25953]: Deploying daemon osd.1 on smithi078 2024-09-17T09:32:29.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:29 smithi136 ceph-mon[25953]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:29.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:29 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:31.744 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:31 smithi078 ceph-mon[23689]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:31.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:31 smithi136 ceph-mon[25953]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:33.766 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:33 smithi078 ceph-mon[23689]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:33.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:33 smithi136 ceph-mon[25953]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:35.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:35 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:35.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:35 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:35.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:35 smithi078 ceph-mon[23689]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:35.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:35 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:35.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:35 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:32:35.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:35 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:32:35.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:35 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:32:35.719 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:35 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:35.719 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:35 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:35.719 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:35 smithi136 ceph-mon[25953]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:35.719 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:35 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:35.720 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:35 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:32:35.720 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:35 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:32:35.720 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:35 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:32:37.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:37 smithi078 ceph-mon[23689]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:37.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:37 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:37.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:37 smithi136 ceph-mon[25953]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:37.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:37 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:38.315 INFO:teuthology.orchestra.run.smithi078.stdout:Created osd(s) 1 on host 'smithi078' 2024-09-17T09:32:38.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:38 smithi078 ceph-mon[23689]: from='osd.1 [v2:172.21.15.78:6810/198469681,v1:172.21.15.78:6811/198469681]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-17T09:32:38.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:38 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:38.893 DEBUG:teuthology.orchestra.run.smithi078:osd.1> sudo journalctl -f -n 0 -u ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.1.service 2024-09-17T09:32:38.895 INFO:tasks.cephadm:Deploying osd.2 on smithi078 with /dev/vg_nvme/lv_2... 2024-09-17T09:32:38.895 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-17T09:32:38.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:38 smithi136 ceph-mon[25953]: from='osd.1 [v2:172.21.15.78:6810/198469681,v1:172.21.15.78:6811/198469681]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-17T09:32:38.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:38 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:39.818 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:39 smithi078 ceph-mon[23689]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:39.819 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:39 smithi078 ceph-mon[23689]: from='osd.1 [v2:172.21.15.78:6810/198469681,v1:172.21.15.78:6811/198469681]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-17T09:32:39.819 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:39 smithi078 ceph-mon[23689]: osdmap e12: 2 total, 1 up, 2 in 2024-09-17T09:32:39.819 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:39 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:32:39.819 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:39 smithi078 ceph-mon[23689]: from='osd.1 [v2:172.21.15.78:6810/198469681,v1:172.21.15.78:6811/198469681]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-09-17T09:32:39.819 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:32:39 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1[38202]: 2024-09-17T09:32:39.539+0000 7efebf993700 -1 osd.1 0 waiting for initial osdmap 2024-09-17T09:32:39.819 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:32:39 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1[38202]: 2024-09-17T09:32:39.543+0000 7efeb9b29700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T09:32:39.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:39 smithi136 ceph-mon[25953]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:39.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:39 smithi136 ceph-mon[25953]: from='osd.1 [v2:172.21.15.78:6810/198469681,v1:172.21.15.78:6811/198469681]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-17T09:32:39.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:39 smithi136 ceph-mon[25953]: osdmap e12: 2 total, 1 up, 2 in 2024-09-17T09:32:39.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:39 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:32:39.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:39 smithi136 ceph-mon[25953]: from='osd.1 [v2:172.21.15.78:6810/198469681,v1:172.21.15.78:6811/198469681]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-09-17T09:32:40.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:40 smithi078 ceph-mon[23689]: from='osd.1 [v2:172.21.15.78:6810/198469681,v1:172.21.15.78:6811/198469681]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-09-17T09:32:40.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:40 smithi078 ceph-mon[23689]: osdmap e13: 2 total, 1 up, 2 in 2024-09-17T09:32:40.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:40 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:32:40.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:40 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:32:40.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:40 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:32:40.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:40 smithi136 ceph-mon[25953]: from='osd.1 [v2:172.21.15.78:6810/198469681,v1:172.21.15.78:6811/198469681]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-09-17T09:32:40.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:40 smithi136 ceph-mon[25953]: osdmap e13: 2 total, 1 up, 2 in 2024-09-17T09:32:40.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:40 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:32:40.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:40 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:32:40.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:40 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:32:41.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:41 smithi078 ceph-mon[23689]: purged_snaps scrub starts 2024-09-17T09:32:41.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:41 smithi078 ceph-mon[23689]: purged_snaps scrub ok 2024-09-17T09:32:41.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:41 smithi078 ceph-mon[23689]: pgmap v54: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:41.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:41 smithi078 ceph-mon[23689]: osd.1 [v2:172.21.15.78:6810/198469681,v1:172.21.15.78:6811/198469681] boot 2024-09-17T09:32:41.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:41 smithi078 ceph-mon[23689]: osdmap e14: 2 total, 2 up, 2 in 2024-09-17T09:32:41.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:41 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:32:41.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:41 smithi136 ceph-mon[25953]: purged_snaps scrub starts 2024-09-17T09:32:41.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:41 smithi136 ceph-mon[25953]: purged_snaps scrub ok 2024-09-17T09:32:41.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:41 smithi136 ceph-mon[25953]: pgmap v54: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T09:32:41.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:41 smithi136 ceph-mon[25953]: osd.1 [v2:172.21.15.78:6810/198469681,v1:172.21.15.78:6811/198469681] boot 2024-09-17T09:32:41.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:41 smithi136 ceph-mon[25953]: osdmap e14: 2 total, 2 up, 2 in 2024-09-17T09:32:41.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:41 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:32:42.633 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:42 smithi078 ceph-mon[23689]: osdmap e15: 2 total, 2 up, 2 in 2024-09-17T09:32:42.638 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-17T09:32:42.659 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi078:vg_nvme/lv_2 2024-09-17T09:32:42.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:42 smithi136 ceph-mon[25953]: osdmap e15: 2 total, 2 up, 2 in 2024-09-17T09:32:43.756 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:43 smithi078 ceph-mon[23689]: pgmap v57: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-17T09:32:43.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:43 smithi136 ceph-mon[25953]: pgmap v57: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-17T09:32:45.835 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:45 smithi078 ceph-mon[23689]: pgmap v58: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-17T09:32:45.835 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:45 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:45.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:45 smithi136 ceph-mon[25953]: pgmap v58: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-17T09:32:45.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:45 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:46.805 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:46 smithi078 ceph-mon[23689]: from='client.14274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:32:46.805 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:46 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T09:32:46.805 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:46 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T09:32:46.805 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:46 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:32:46.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:46 smithi136 ceph-mon[25953]: from='client.14274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:32:46.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:46 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T09:32:46.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:46 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T09:32:46.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:46 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:32:47.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:47 smithi078 ceph-mon[23689]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T09:32:47.945 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:47 smithi136 ceph-mon[25953]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T09:32:49.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:49.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:32:49.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:32:49.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:49 smithi078 ceph-mon[23689]: Adjusting osd_memory_target on smithi078 to 6432M 2024-09-17T09:32:49.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:49.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:49.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:32:49.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:32:49.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:32:49.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:49 smithi078 ceph-mon[23689]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T09:32:49.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:49.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:32:49.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:32:49.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:49 smithi136 ceph-mon[25953]: Adjusting osd_memory_target on smithi078 to 6432M 2024-09-17T09:32:49.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:49.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:49.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:32:49.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:32:49.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:32:49.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:49 smithi136 ceph-mon[25953]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T09:32:50.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:50 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/1301719428' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fe89957a-ef6e-4208-9fe8-6f1078511f26"}]: dispatch 2024-09-17T09:32:50.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:50 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/1301719428' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fe89957a-ef6e-4208-9fe8-6f1078511f26"}]': finished 2024-09-17T09:32:50.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:50 smithi078 ceph-mon[23689]: osdmap e16: 3 total, 2 up, 3 in 2024-09-17T09:32:50.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:50 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T09:32:50.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:50 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/1301719428' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fe89957a-ef6e-4208-9fe8-6f1078511f26"}]: dispatch 2024-09-17T09:32:50.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:50 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/1301719428' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fe89957a-ef6e-4208-9fe8-6f1078511f26"}]': finished 2024-09-17T09:32:50.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:50 smithi136 ceph-mon[25953]: osdmap e16: 3 total, 2 up, 3 in 2024-09-17T09:32:50.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:50 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T09:32:51.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:51 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/2958671407' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T09:32:51.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:51 smithi078 ceph-mon[23689]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T09:32:51.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:51 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/2958671407' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T09:32:51.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:51 smithi136 ceph-mon[25953]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T09:32:53.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:53 smithi078 ceph-mon[23689]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T09:32:53.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:53 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:53.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:53 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:53.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:53 smithi136 ceph-mon[25953]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T09:32:53.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:53 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:53.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:53 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:32:54.656 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:54 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:32:54.657 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:54 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/trash_purge_schedule"}]: dispatch 2024-09-17T09:32:54.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:54 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:32:54.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:54 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/trash_purge_schedule"}]: dispatch 2024-09-17T09:32:55.730 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:55 smithi078 ceph-mon[23689]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T09:32:55.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:55 smithi136 ceph-mon[25953]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T09:32:56.783 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:56 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T09:32:56.783 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:56 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:32:56.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:56 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T09:32:56.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:56 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:32:57.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:57 smithi136 ceph-mon[25953]: Deploying daemon osd.2 on smithi078 2024-09-17T09:32:57.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:57 smithi136 ceph-mon[25953]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T09:32:57.782 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:57 smithi078 ceph-mon[23689]: Deploying daemon osd.2 on smithi078 2024-09-17T09:32:57.782 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:57 smithi078 ceph-mon[23689]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T09:32:59.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:32:59 smithi078 ceph-mon[23689]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T09:32:59.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:32:59 smithi136 ceph-mon[25953]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T09:33:01.782 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:01 smithi078 ceph-mon[23689]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T09:33:01.782 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:01 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:01.782 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:01 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:33:01.783 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:01 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:33:01.783 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:01 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:33:01.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:01 smithi136 ceph-mon[25953]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T09:33:01.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:01 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:01.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:01 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:33:01.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:01 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:33:01.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:01 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:33:03.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:03 smithi078 ceph-mon[23689]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T09:33:03.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:03 smithi136 ceph-mon[25953]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T09:33:04.429 INFO:teuthology.orchestra.run.smithi078.stdout:Created osd(s) 2 on host 'smithi078' 2024-09-17T09:33:04.773 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:04 smithi078 ceph-mon[23689]: from='osd.2 [v2:172.21.15.78:6818/1349073215,v1:172.21.15.78:6819/1349073215]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T09:33:04.773 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:04 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:04.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:04 smithi136 ceph-mon[25953]: from='osd.2 [v2:172.21.15.78:6818/1349073215,v1:172.21.15.78:6819/1349073215]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T09:33:04.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:04 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:05.100 DEBUG:teuthology.orchestra.run.smithi078:osd.2> sudo journalctl -f -n 0 -u ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.2.service 2024-09-17T09:33:05.103 INFO:tasks.cephadm:Deploying osd.3 on smithi078 with /dev/vg_nvme/lv_1... 2024-09-17T09:33:05.103 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-17T09:33:05.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:05 smithi078 ceph-mon[23689]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T09:33:05.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:05 smithi078 ceph-mon[23689]: from='osd.2 [v2:172.21.15.78:6818/1349073215,v1:172.21.15.78:6819/1349073215]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-17T09:33:05.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:05 smithi078 ceph-mon[23689]: osdmap e17: 3 total, 2 up, 3 in 2024-09-17T09:33:05.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:05 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T09:33:05.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:05 smithi078 ceph-mon[23689]: from='osd.2 [v2:172.21.15.78:6818/1349073215,v1:172.21.15.78:6819/1349073215]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-09-17T09:33:05.727 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:33:05 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2[41869]: 2024-09-17T09:33:05.564+0000 7feda3619700 -1 osd.2 0 waiting for initial osdmap 2024-09-17T09:33:05.727 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:33:05 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2[41869]: 2024-09-17T09:33:05.567+0000 7fed9ffb4700 -1 osd.2 18 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T09:33:05.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:05 smithi136 ceph-mon[25953]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T09:33:05.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:05 smithi136 ceph-mon[25953]: from='osd.2 [v2:172.21.15.78:6818/1349073215,v1:172.21.15.78:6819/1349073215]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-17T09:33:05.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:05 smithi136 ceph-mon[25953]: osdmap e17: 3 total, 2 up, 3 in 2024-09-17T09:33:05.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:05 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T09:33:05.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:05 smithi136 ceph-mon[25953]: from='osd.2 [v2:172.21.15.78:6818/1349073215,v1:172.21.15.78:6819/1349073215]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-09-17T09:33:06.832 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:06 smithi078 ceph-mon[23689]: from='osd.2 [v2:172.21.15.78:6818/1349073215,v1:172.21.15.78:6819/1349073215]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-09-17T09:33:06.832 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:06 smithi078 ceph-mon[23689]: osdmap e18: 3 total, 2 up, 3 in 2024-09-17T09:33:06.832 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:06 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T09:33:06.832 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:06 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T09:33:06.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:06 smithi136 ceph-mon[25953]: from='osd.2 [v2:172.21.15.78:6818/1349073215,v1:172.21.15.78:6819/1349073215]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-09-17T09:33:06.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:06 smithi136 ceph-mon[25953]: osdmap e18: 3 total, 2 up, 3 in 2024-09-17T09:33:06.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:06 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T09:33:06.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:06 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T09:33:07.797 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:07 smithi078 ceph-mon[23689]: purged_snaps scrub starts 2024-09-17T09:33:07.797 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:07 smithi078 ceph-mon[23689]: purged_snaps scrub ok 2024-09-17T09:33:07.797 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:07 smithi078 ceph-mon[23689]: pgmap v72: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T09:33:07.797 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:07 smithi078 ceph-mon[23689]: osd.2 [v2:172.21.15.78:6818/1349073215,v1:172.21.15.78:6819/1349073215] boot 2024-09-17T09:33:07.797 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:07 smithi078 ceph-mon[23689]: osdmap e19: 3 total, 3 up, 3 in 2024-09-17T09:33:07.797 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:07 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T09:33:07.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:07 smithi136 ceph-mon[25953]: purged_snaps scrub starts 2024-09-17T09:33:07.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:07 smithi136 ceph-mon[25953]: purged_snaps scrub ok 2024-09-17T09:33:07.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:07 smithi136 ceph-mon[25953]: pgmap v72: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T09:33:07.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:07 smithi136 ceph-mon[25953]: osd.2 [v2:172.21.15.78:6818/1349073215,v1:172.21.15.78:6819/1349073215] boot 2024-09-17T09:33:07.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:07 smithi136 ceph-mon[25953]: osdmap e19: 3 total, 3 up, 3 in 2024-09-17T09:33:07.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:07 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T09:33:08.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:08 smithi078 ceph-mon[23689]: osdmap e20: 3 total, 3 up, 3 in 2024-09-17T09:33:08.882 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-17T09:33:08.901 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi078:vg_nvme/lv_1 2024-09-17T09:33:09.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:08 smithi136 ceph-mon[25953]: osdmap e20: 3 total, 3 up, 3 in 2024-09-17T09:33:10.085 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:09 smithi078 ceph-mon[23689]: pgmap v75: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T09:33:10.086 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:09 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-17T09:33:10.086 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:09 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-17T09:33:10.086 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:09 smithi078 ceph-mon[23689]: osdmap e21: 3 total, 3 up, 3 in 2024-09-17T09:33:10.086 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:09 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-17T09:33:10.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:09 smithi136 ceph-mon[25953]: pgmap v75: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T09:33:10.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:09 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-17T09:33:10.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:09 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-17T09:33:10.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:09 smithi136 ceph-mon[25953]: osdmap e21: 3 total, 3 up, 3 in 2024-09-17T09:33:10.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:09 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-17T09:33:11.813 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:11 smithi078 ceph-mon[23689]: pgmap v77: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T09:33:11.813 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:11 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-17T09:33:11.813 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:11 smithi078 ceph-mon[23689]: osdmap e22: 3 total, 3 up, 3 in 2024-09-17T09:33:11.813 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:11 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:11.814 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:33:11 smithi078 sudo[43386]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T09:33:11.814 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:33:11 smithi078 sudo[43386]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T09:33:11.814 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:33:11 smithi078 sudo[43386]: pam_unix(sudo:session): session closed for user root 2024-09-17T09:33:11.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:11 smithi136 ceph-mon[25953]: pgmap v77: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T09:33:11.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:11 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-17T09:33:11.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:11 smithi136 ceph-mon[25953]: osdmap e22: 3 total, 3 up, 3 in 2024-09-17T09:33:11.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:11 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:12.096 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:33:11 smithi078 sudo[43443]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T09:33:12.096 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:33:11 smithi078 sudo[43443]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T09:33:12.096 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:33:11 smithi078 sudo[43443]: pam_unix(sudo:session): session closed for user root 2024-09-17T09:33:12.404 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:33:12 smithi078 sudo[43464]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T09:33:12.404 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:33:12 smithi078 sudo[43464]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T09:33:12.404 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:33:12 smithi078 sudo[43464]: pam_unix(sudo:session): session closed for user root 2024-09-17T09:33:12.667 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:33:12 smithi078 sudo[43468]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T09:33:12.667 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:33:12 smithi078 sudo[43468]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T09:33:12.667 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:33:12 smithi078 sudo[43468]: pam_unix(sudo:session): session closed for user root 2024-09-17T09:33:12.667 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:33:12 smithi078 sudo[43473]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T09:33:12.667 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:33:12 smithi078 sudo[43473]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T09:33:12.667 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:33:12 smithi078 sudo[43473]: pam_unix(sudo:session): session closed for user root 2024-09-17T09:33:12.668 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:12 smithi078 ceph-mon[23689]: from='client.14296 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:33:12.668 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:12 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T09:33:12.668 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:12 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T09:33:12.668 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:12 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:33:12.668 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:12 smithi078 ceph-mon[23689]: osdmap e23: 3 total, 3 up, 3 in 2024-09-17T09:33:12.668 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:12 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:12.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:12 smithi136 ceph-mon[25953]: from='client.14296 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:33:12.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:12 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T09:33:12.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:12 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T09:33:12.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:12 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:33:12.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:12 smithi136 ceph-mon[25953]: osdmap e23: 3 total, 3 up, 3 in 2024-09-17T09:33:12.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:12 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:13.011 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:33:12 smithi078 sudo[43492]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T09:33:13.011 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:33:12 smithi078 sudo[43492]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T09:33:13.011 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:33:12 smithi078 sudo[43492]: pam_unix(sudo:session): session closed for user root 2024-09-17T09:33:13.263 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:13 smithi078 sudo[43518]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T09:33:13.264 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:13 smithi078 sudo[43518]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T09:33:13.264 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:13 smithi078 sudo[43518]: pam_unix(sudo:session): session closed for user root 2024-09-17T09:33:13.514 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:13 smithi078 sudo[43534]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T09:33:13.514 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:13 smithi078 sudo[43534]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T09:33:13.514 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:13 smithi078 sudo[43534]: pam_unix(sudo:session): session closed for user root 2024-09-17T09:33:13.919 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:13 smithi136 sudo[28287]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T09:33:13.920 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:13 smithi136 sudo[28287]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T09:33:13.920 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:13 smithi136 sudo[28287]: pam_unix(sudo:session): session closed for user root 2024-09-17T09:33:13.920 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:13 smithi136 sudo[28290]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T09:33:13.920 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:13 smithi136 sudo[28290]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T09:33:13.920 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:13 smithi136 sudo[28290]: pam_unix(sudo:session): session closed for user root 2024-09-17T09:33:14.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:13 smithi078 ceph-mon[23689]: pgmap v80: 1 pgs: 1 active+clean; 449 KiB data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T09:33:14.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:13 smithi078 ceph-mon[23689]: mgrmap e19: smithi078.rsqofi(active, since 2m), standbys: smithi136.hicmlm 2024-09-17T09:33:14.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:13 smithi078 ceph-mon[23689]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T09:33:14.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:13 smithi078 ceph-mon[23689]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T09:33:14.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:13 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-17T09:33:14.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:13 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata", "id": "smithi136"}]: dispatch 2024-09-17T09:33:14.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:13 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-17T09:33:14.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:13 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata", "id": "smithi136"}]: dispatch 2024-09-17T09:33:14.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:13 smithi136 ceph-mon[25953]: pgmap v80: 1 pgs: 1 active+clean; 449 KiB data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T09:33:14.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:13 smithi136 ceph-mon[25953]: mgrmap e19: smithi078.rsqofi(active, since 2m), standbys: smithi136.hicmlm 2024-09-17T09:33:14.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:13 smithi136 ceph-mon[25953]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T09:33:14.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:13 smithi136 ceph-mon[25953]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T09:33:14.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:13 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-17T09:33:14.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:13 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata", "id": "smithi136"}]: dispatch 2024-09-17T09:33:14.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:13 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-17T09:33:14.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:13 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata", "id": "smithi136"}]: dispatch 2024-09-17T09:33:15.216 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:14 smithi078 ceph-mon[23689]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T09:33:15.216 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:14 smithi078 ceph-mon[23689]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T09:33:15.216 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:14 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:15.216 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:14 smithi078 ceph-mon[23689]: pgmap v81: 1 pgs: 1 active+clean; 449 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-09-17T09:33:15.216 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:14 smithi078 ceph-mon[23689]: Detected new or changed devices on smithi078 2024-09-17T09:33:15.216 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:14 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:15.216 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:14 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:33:15.216 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:14 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:33:15.216 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:14 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:33:15.217 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:14 smithi078 ceph-mon[23689]: Adjusting osd_memory_target on smithi078 to 4288M 2024-09-17T09:33:15.217 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:14 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:15.217 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:14 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:15.217 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:14 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:33:15.217 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:14 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:33:15.217 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:14 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:33:15.217 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:14 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:15.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:14 smithi136 ceph-mon[25953]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T09:33:15.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:14 smithi136 ceph-mon[25953]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T09:33:15.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:14 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:15.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:14 smithi136 ceph-mon[25953]: pgmap v81: 1 pgs: 1 active+clean; 449 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-09-17T09:33:15.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:14 smithi136 ceph-mon[25953]: Detected new or changed devices on smithi078 2024-09-17T09:33:15.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:14 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:15.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:14 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:33:15.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:14 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:33:15.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:14 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:33:15.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:14 smithi136 ceph-mon[25953]: Adjusting osd_memory_target on smithi078 to 4288M 2024-09-17T09:33:15.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:14 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:15.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:14 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:15.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:14 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:33:15.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:14 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:33:15.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:14 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:33:15.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:14 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:16.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:16 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/1566254487' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b4106928-6750-4445-bef2-d5c216e9ae7c"}]: dispatch 2024-09-17T09:33:16.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:16 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/1566254487' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b4106928-6750-4445-bef2-d5c216e9ae7c"}]': finished 2024-09-17T09:33:16.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:16 smithi136 ceph-mon[25953]: osdmap e24: 4 total, 3 up, 4 in 2024-09-17T09:33:16.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:16 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:33:16.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:16 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/1566254487' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b4106928-6750-4445-bef2-d5c216e9ae7c"}]: dispatch 2024-09-17T09:33:16.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:16 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/1566254487' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b4106928-6750-4445-bef2-d5c216e9ae7c"}]': finished 2024-09-17T09:33:16.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:16 smithi078 ceph-mon[23689]: osdmap e24: 4 total, 3 up, 4 in 2024-09-17T09:33:16.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:16 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:33:17.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:17 smithi136 ceph-mon[25953]: pgmap v83: 1 pgs: 1 active+clean; 449 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-09-17T09:33:17.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:17 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/1188349366' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T09:33:17.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:17 smithi078 ceph-mon[23689]: pgmap v83: 1 pgs: 1 active+clean; 449 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-09-17T09:33:17.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:17 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/1188349366' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T09:33:19.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:19 smithi078 ceph-mon[23689]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 4.0 KiB/s rd, 195 KiB/s wr, 13 op/s 2024-09-17T09:33:19.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:19 smithi136 ceph-mon[25953]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 4.0 KiB/s rd, 195 KiB/s wr, 13 op/s 2024-09-17T09:33:21.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:21 smithi078 ceph-mon[23689]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.1 KiB/s rd, 150 KiB/s wr, 10 op/s 2024-09-17T09:33:21.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:21 smithi136 ceph-mon[25953]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.1 KiB/s rd, 150 KiB/s wr, 10 op/s 2024-09-17T09:33:22.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:22 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T09:33:22.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:22 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:33:22.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:22 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T09:33:22.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:22 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:33:23.830 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:23 smithi078 ceph-mon[23689]: Deploying daemon osd.3 on smithi078 2024-09-17T09:33:23.830 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:23 smithi078 ceph-mon[23689]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-09-17T09:33:23.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:23 smithi136 ceph-mon[25953]: Deploying daemon osd.3 on smithi078 2024-09-17T09:33:23.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:23 smithi136 ceph-mon[25953]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-09-17T09:33:25.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:25 smithi078 ceph-mon[23689]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-09-17T09:33:25.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:25 smithi136 ceph-mon[25953]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-09-17T09:33:27.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:27 smithi078 ceph-mon[23689]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.6 KiB/s rd, 125 KiB/s wr, 8 op/s 2024-09-17T09:33:27.943 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:27 smithi136 ceph-mon[25953]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.6 KiB/s rd, 125 KiB/s wr, 8 op/s 2024-09-17T09:33:28.928 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:28 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:28.929 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:28 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:33:28.929 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:28 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:33:28.929 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:28 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:33:28.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:28 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:28.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:28 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:33:28.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:28 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:33:28.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:28 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:33:29.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:29 smithi136 ceph-mon[25953]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 109 KiB/s wr, 7 op/s 2024-09-17T09:33:30.026 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:29 smithi078 ceph-mon[23689]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 109 KiB/s wr, 7 op/s 2024-09-17T09:33:30.669 INFO:teuthology.orchestra.run.smithi078.stdout:Created osd(s) 3 on host 'smithi078' 2024-09-17T09:33:31.292 DEBUG:teuthology.orchestra.run.smithi078:osd.3> sudo journalctl -f -n 0 -u ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.3.service 2024-09-17T09:33:31.295 INFO:tasks.cephadm:Deploying osd.4 on smithi136 with /dev/vg_nvme/lv_4... 2024-09-17T09:33:31.295 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-17T09:33:31.910 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:31 smithi136 ceph-mon[25953]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T09:33:31.910 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:31 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:31.910 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:31 smithi136 ceph-mon[25953]: from='osd.3 [v2:172.21.15.78:6826/3175336990,v1:172.21.15.78:6827/3175336990]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-17T09:33:32.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:31 smithi078 ceph-mon[23689]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T09:33:32.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:31 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:32.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:31 smithi078 ceph-mon[23689]: from='osd.3 [v2:172.21.15.78:6826/3175336990,v1:172.21.15.78:6827/3175336990]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-17T09:33:32.702 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:32 smithi136 ceph-mon[25953]: from='osd.3 [v2:172.21.15.78:6826/3175336990,v1:172.21.15.78:6827/3175336990]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-17T09:33:32.703 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:32 smithi136 ceph-mon[25953]: osdmap e25: 4 total, 3 up, 4 in 2024-09-17T09:33:32.703 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:32 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:33:32.703 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:32 smithi136 ceph-mon[25953]: from='osd.3 [v2:172.21.15.78:6826/3175336990,v1:172.21.15.78:6827/3175336990]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-09-17T09:33:33.008 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:32 smithi078 ceph-mon[23689]: from='osd.3 [v2:172.21.15.78:6826/3175336990,v1:172.21.15.78:6827/3175336990]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-17T09:33:33.008 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:32 smithi078 ceph-mon[23689]: osdmap e25: 4 total, 3 up, 4 in 2024-09-17T09:33:33.008 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:32 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:33:33.008 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:32 smithi078 ceph-mon[23689]: from='osd.3 [v2:172.21.15.78:6826/3175336990,v1:172.21.15.78:6827/3175336990]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-09-17T09:33:33.008 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:33:32 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3[45123]: 2024-09-17T09:33:32.675+0000 7fe9530f3700 -1 osd.3 0 waiting for initial osdmap 2024-09-17T09:33:33.008 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:33:32 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3[45123]: 2024-09-17T09:33:32.687+0000 7fe94ba86700 -1 osd.3 26 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T09:33:33.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:33 smithi136 ceph-mon[25953]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T09:33:33.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:33 smithi136 ceph-mon[25953]: from='osd.3 [v2:172.21.15.78:6826/3175336990,v1:172.21.15.78:6827/3175336990]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-09-17T09:33:33.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:33 smithi136 ceph-mon[25953]: osdmap e26: 4 total, 3 up, 4 in 2024-09-17T09:33:33.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:33 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:33:33.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:33 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:33:33.833 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-17T09:33:33.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:33 smithi078 ceph-mon[23689]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T09:33:33.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:33 smithi078 ceph-mon[23689]: from='osd.3 [v2:172.21.15.78:6826/3175336990,v1:172.21.15.78:6827/3175336990]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-09-17T09:33:33.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:33 smithi078 ceph-mon[23689]: osdmap e26: 4 total, 3 up, 4 in 2024-09-17T09:33:33.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:33 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:33:33.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:33 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:33:33.853 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi136:vg_nvme/lv_4 2024-09-17T09:33:34.833 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:34 smithi078 ceph-mon[23689]: purged_snaps scrub starts 2024-09-17T09:33:34.833 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:34 smithi078 ceph-mon[23689]: purged_snaps scrub ok 2024-09-17T09:33:34.833 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:34 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:33:34.833 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:34 smithi078 ceph-mon[23689]: osd.3 [v2:172.21.15.78:6826/3175336990,v1:172.21.15.78:6827/3175336990] boot 2024-09-17T09:33:34.833 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:34 smithi078 ceph-mon[23689]: osdmap e27: 4 total, 4 up, 4 in 2024-09-17T09:33:34.834 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:34 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:33:34.834 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:34 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:34.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:34 smithi136 ceph-mon[25953]: purged_snaps scrub starts 2024-09-17T09:33:34.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:34 smithi136 ceph-mon[25953]: purged_snaps scrub ok 2024-09-17T09:33:34.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:34 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:33:34.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:34 smithi136 ceph-mon[25953]: osd.3 [v2:172.21.15.78:6826/3175336990,v1:172.21.15.78:6827/3175336990] boot 2024-09-17T09:33:34.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:34 smithi136 ceph-mon[25953]: osdmap e27: 4 total, 4 up, 4 in 2024-09-17T09:33:34.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:34 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:33:34.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:34 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:35.924 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:35 smithi078 ceph-mon[23689]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T09:33:35.924 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:35 smithi078 ceph-mon[23689]: osdmap e28: 4 total, 4 up, 4 in 2024-09-17T09:33:35.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:35 smithi136 ceph-mon[25953]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T09:33:35.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:35 smithi136 ceph-mon[25953]: osdmap e28: 4 total, 4 up, 4 in 2024-09-17T09:33:36.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:36 smithi136 ceph-mon[25953]: from='client.24141 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:33:36.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:36 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T09:33:36.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:36 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T09:33:36.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:36 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:33:37.088 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:36 smithi078 ceph-mon[23689]: from='client.24141 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:33:37.088 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:36 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T09:33:37.089 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:36 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T09:33:37.089 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:36 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:33:37.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:37 smithi136 ceph-mon[25953]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T09:33:37.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:37 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:37.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:37 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:33:37.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:37 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:33:37.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:37 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:33:37.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:37 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:33:37.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:37 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:37.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:37 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:37.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:37 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:33:37.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:37 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:33:37.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:37 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:33:37.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:37 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:38.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:37 smithi078 ceph-mon[23689]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T09:33:38.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:37 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:38.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:37 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:33:38.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:37 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:33:38.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:37 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:33:38.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:37 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:33:38.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:37 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:38.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:37 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:38.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:37 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:33:38.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:37 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:33:38.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:37 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:33:38.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:37 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:38.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:38 smithi136 ceph-mon[25953]: Detected new or changed devices on smithi078 2024-09-17T09:33:38.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:38 smithi136 ceph-mon[25953]: Adjusting osd_memory_target on smithi078 to 3216M 2024-09-17T09:33:38.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:38 smithi136 ceph-mon[25953]: from='client.? 172.21.15.136:0/1342614263' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3334c021-ced7-4aff-9ee4-77e6ef91e87c"}]: dispatch 2024-09-17T09:33:38.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:38 smithi136 ceph-mon[25953]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3334c021-ced7-4aff-9ee4-77e6ef91e87c"}]: dispatch 2024-09-17T09:33:38.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:38 smithi136 ceph-mon[25953]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3334c021-ced7-4aff-9ee4-77e6ef91e87c"}]': finished 2024-09-17T09:33:38.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:38 smithi136 ceph-mon[25953]: osdmap e29: 5 total, 4 up, 5 in 2024-09-17T09:33:38.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:38 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:33:38.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:38 smithi136 ceph-mon[25953]: from='client.? 172.21.15.136:0/4233652368' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T09:33:39.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:38 smithi078 ceph-mon[23689]: Detected new or changed devices on smithi078 2024-09-17T09:33:39.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:38 smithi078 ceph-mon[23689]: Adjusting osd_memory_target on smithi078 to 3216M 2024-09-17T09:33:39.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:38 smithi078 ceph-mon[23689]: from='client.? 172.21.15.136:0/1342614263' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3334c021-ced7-4aff-9ee4-77e6ef91e87c"}]: dispatch 2024-09-17T09:33:39.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:38 smithi078 ceph-mon[23689]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3334c021-ced7-4aff-9ee4-77e6ef91e87c"}]: dispatch 2024-09-17T09:33:39.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:38 smithi078 ceph-mon[23689]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3334c021-ced7-4aff-9ee4-77e6ef91e87c"}]': finished 2024-09-17T09:33:39.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:38 smithi078 ceph-mon[23689]: osdmap e29: 5 total, 4 up, 5 in 2024-09-17T09:33:39.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:38 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:33:39.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:38 smithi078 ceph-mon[23689]: from='client.? 172.21.15.136:0/4233652368' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T09:33:39.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:39 smithi136 ceph-mon[25953]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T09:33:40.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:39 smithi078 ceph-mon[23689]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T09:33:41.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:41 smithi136 ceph-mon[25953]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T09:33:42.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:41 smithi078 ceph-mon[23689]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T09:33:43.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:43 smithi136 ceph-mon[25953]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T09:33:44.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:43 smithi078 ceph-mon[23689]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T09:33:44.972 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:44 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T09:33:44.972 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:44 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:33:45.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:44 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T09:33:45.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:44 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:33:45.896 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:45 smithi136 ceph-mon[25953]: Deploying daemon osd.4 on smithi136 2024-09-17T09:33:45.897 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:45 smithi136 ceph-mon[25953]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T09:33:46.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:45 smithi078 ceph-mon[23689]: Deploying daemon osd.4 on smithi136 2024-09-17T09:33:46.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:45 smithi078 ceph-mon[23689]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T09:33:47.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:47 smithi136 ceph-mon[25953]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T09:33:48.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:47 smithi078 ceph-mon[23689]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T09:33:49.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:49 smithi136 ceph-mon[25953]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T09:33:49.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:49.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:33:49.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:33:49.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:33:50.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:49 smithi078 ceph-mon[23689]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T09:33:50.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:50.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:33:50.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:33:50.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:33:51.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:51 smithi136 ceph-mon[25953]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T09:33:52.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:51 smithi078 ceph-mon[23689]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T09:33:52.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:52 smithi136 ceph-mon[25953]: from='osd.4 [v2:172.21.15.136:6800/2032814238,v1:172.21.15.136:6801/2032814238]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T09:33:52.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:52 smithi136 ceph-mon[25953]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T09:33:53.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:52 smithi078 ceph-mon[23689]: from='osd.4 [v2:172.21.15.136:6800/2032814238,v1:172.21.15.136:6801/2032814238]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T09:33:53.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:52 smithi078 ceph-mon[23689]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T09:33:53.392 INFO:teuthology.orchestra.run.smithi136.stdout:Created osd(s) 4 on host 'smithi136' 2024-09-17T09:33:54.057 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:53 smithi136 ceph-mon[25953]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T09:33:54.057 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:53 smithi136 ceph-mon[25953]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-17T09:33:54.058 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:53 smithi136 ceph-mon[25953]: osdmap e30: 5 total, 4 up, 5 in 2024-09-17T09:33:54.058 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:53 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:33:54.058 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:53 smithi136 ceph-mon[25953]: from='osd.4 [v2:172.21.15.136:6800/2032814238,v1:172.21.15.136:6801/2032814238]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-17T09:33:54.058 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:53 smithi136 ceph-mon[25953]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-17T09:33:54.058 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:53 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:54.058 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:53 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:54.058 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:53 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:54.058 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:53 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:33:54.058 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:53 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:33:54.058 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:53 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:33:54.087 DEBUG:teuthology.orchestra.run.smithi136:osd.4> sudo journalctl -f -n 0 -u ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.4.service 2024-09-17T09:33:54.089 INFO:tasks.cephadm:Deploying osd.5 on smithi136 with /dev/vg_nvme/lv_3... 2024-09-17T09:33:54.089 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-17T09:33:54.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:53 smithi078 ceph-mon[23689]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T09:33:54.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:53 smithi078 ceph-mon[23689]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-17T09:33:54.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:53 smithi078 ceph-mon[23689]: osdmap e30: 5 total, 4 up, 5 in 2024-09-17T09:33:54.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:53 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:33:54.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:53 smithi078 ceph-mon[23689]: from='osd.4 [v2:172.21.15.136:6800/2032814238,v1:172.21.15.136:6801/2032814238]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-17T09:33:54.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:53 smithi078 ceph-mon[23689]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-17T09:33:54.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:53 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:54.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:53 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:54.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:53 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:54.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:53 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:33:54.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:53 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:33:54.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:53 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:33:54.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:54 smithi078 ceph-mon[23689]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2024-09-17T09:33:54.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:54 smithi078 ceph-mon[23689]: osdmap e31: 5 total, 4 up, 5 in 2024-09-17T09:33:54.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:54 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:33:54.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:54 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:33:54.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:54 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/trash_purge_schedule"}]: dispatch 2024-09-17T09:33:54.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:54 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:33:54.868 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:54 smithi136 ceph-mon[25953]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2024-09-17T09:33:54.868 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:54 smithi136 ceph-mon[25953]: osdmap e31: 5 total, 4 up, 5 in 2024-09-17T09:33:54.869 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:54 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:33:54.869 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:54 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:33:54.869 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:54 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/trash_purge_schedule"}]: dispatch 2024-09-17T09:33:54.869 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:54 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:33:56.042 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:55 smithi136 ceph-mon[25953]: purged_snaps scrub starts 2024-09-17T09:33:56.042 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:55 smithi136 ceph-mon[25953]: purged_snaps scrub ok 2024-09-17T09:33:56.042 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:55 smithi136 ceph-mon[25953]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T09:33:56.042 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:55 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:33:56.042 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:55 smithi136 ceph-mon[25953]: osd.4 [v2:172.21.15.136:6800/2032814238,v1:172.21.15.136:6801/2032814238] boot 2024-09-17T09:33:56.042 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:55 smithi136 ceph-mon[25953]: osdmap e32: 5 total, 5 up, 5 in 2024-09-17T09:33:56.042 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:55 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:33:56.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:55 smithi078 ceph-mon[23689]: purged_snaps scrub starts 2024-09-17T09:33:56.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:55 smithi078 ceph-mon[23689]: purged_snaps scrub ok 2024-09-17T09:33:56.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:55 smithi078 ceph-mon[23689]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T09:33:56.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:55 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:33:56.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:55 smithi078 ceph-mon[23689]: osd.4 [v2:172.21.15.136:6800/2032814238,v1:172.21.15.136:6801/2032814238] boot 2024-09-17T09:33:56.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:55 smithi078 ceph-mon[23689]: osdmap e32: 5 total, 5 up, 5 in 2024-09-17T09:33:56.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:55 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:33:56.969 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:56 smithi136 ceph-mon[25953]: osdmap e33: 5 total, 5 up, 5 in 2024-09-17T09:33:57.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:56 smithi078 ceph-mon[23689]: osdmap e33: 5 total, 5 up, 5 in 2024-09-17T09:33:57.844 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:57 smithi136 ceph-mon[25953]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-17T09:33:57.845 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:57 smithi136 ceph-mon[25953]: osdmap e34: 5 total, 5 up, 5 in 2024-09-17T09:33:57.845 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:57 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:57.845 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:57 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:33:57.845 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:57 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:57.845 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:57 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:58.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:57 smithi078 ceph-mon[23689]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-17T09:33:58.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:57 smithi078 ceph-mon[23689]: osdmap e34: 5 total, 5 up, 5 in 2024-09-17T09:33:58.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:57 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:58.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:57 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:33:58.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:57 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:58.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:57 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:33:58.930 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:58 smithi136 ceph-mon[25953]: Detected new or changed devices on smithi136 2024-09-17T09:33:58.931 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:58 smithi136 ceph-mon[25953]: Adjusting osd_memory_target on smithi136 to 15936M 2024-09-17T09:33:58.931 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:58 smithi136 ceph-mon[25953]: osdmap e35: 5 total, 5 up, 5 in 2024-09-17T09:33:58.935 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-17T09:33:58.954 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi136:vg_nvme/lv_3 2024-09-17T09:33:59.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:58 smithi078 ceph-mon[23689]: Detected new or changed devices on smithi136 2024-09-17T09:33:59.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:58 smithi078 ceph-mon[23689]: Adjusting osd_memory_target on smithi136 to 15936M 2024-09-17T09:33:59.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:58 smithi078 ceph-mon[23689]: osdmap e35: 5 total, 5 up, 5 in 2024-09-17T09:33:59.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:59 smithi136 ceph-mon[25953]: pgmap v115: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-17T09:33:59.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:33:59 smithi136 ceph-mon[25953]: osdmap e36: 5 total, 5 up, 5 in 2024-09-17T09:34:00.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:59 smithi078 ceph-mon[23689]: pgmap v115: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-17T09:34:00.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:33:59 smithi078 ceph-mon[23689]: osdmap e36: 5 total, 5 up, 5 in 2024-09-17T09:34:01.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:01 smithi136 ceph-mon[25953]: pgmap v117: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T09:34:01.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:01 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T09:34:01.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:01 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T09:34:01.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:01 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:34:02.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:01 smithi078 ceph-mon[23689]: pgmap v117: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T09:34:02.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:01 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T09:34:02.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:01 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T09:34:02.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:01 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:34:03.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:02 smithi078 ceph-mon[23689]: from='client.24159 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:34:03.165 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:02 smithi136 ceph-mon[25953]: from='client.24159 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:34:04.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:03 smithi078 ceph-mon[23689]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-17T09:34:04.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:03 smithi078 ceph-mon[23689]: from='client.? 172.21.15.136:0/556330147' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3b900ce7-186b-45dc-a15c-5b078321c9f4"}]: dispatch 2024-09-17T09:34:04.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:03 smithi078 ceph-mon[23689]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3b900ce7-186b-45dc-a15c-5b078321c9f4"}]: dispatch 2024-09-17T09:34:04.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:03 smithi078 ceph-mon[23689]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3b900ce7-186b-45dc-a15c-5b078321c9f4"}]': finished 2024-09-17T09:34:04.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:03 smithi078 ceph-mon[23689]: osdmap e37: 6 total, 5 up, 6 in 2024-09-17T09:34:04.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:03 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T09:34:04.171 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:03 smithi136 ceph-mon[25953]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-17T09:34:04.172 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:03 smithi136 ceph-mon[25953]: from='client.? 172.21.15.136:0/556330147' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3b900ce7-186b-45dc-a15c-5b078321c9f4"}]: dispatch 2024-09-17T09:34:04.172 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:03 smithi136 ceph-mon[25953]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3b900ce7-186b-45dc-a15c-5b078321c9f4"}]: dispatch 2024-09-17T09:34:04.172 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:03 smithi136 ceph-mon[25953]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3b900ce7-186b-45dc-a15c-5b078321c9f4"}]': finished 2024-09-17T09:34:04.172 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:03 smithi136 ceph-mon[25953]: osdmap e37: 6 total, 5 up, 6 in 2024-09-17T09:34:04.172 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:03 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T09:34:05.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:04 smithi078 ceph-mon[23689]: from='client.? 172.21.15.136:0/3285907693' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T09:34:05.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:04 smithi136 ceph-mon[25953]: from='client.? 172.21.15.136:0/3285907693' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T09:34:06.073 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:05 smithi136 ceph-mon[25953]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-09-17T09:34:06.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:05 smithi078 ceph-mon[23689]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-09-17T09:34:07.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:07 smithi136 ceph-mon[25953]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-17T09:34:08.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:07 smithi078 ceph-mon[23689]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-17T09:34:10.029 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:09 smithi136 ceph-mon[25953]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-09-17T09:34:10.029 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:09 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T09:34:10.029 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:09 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:34:10.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:09 smithi078 ceph-mon[23689]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-09-17T09:34:10.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:09 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T09:34:10.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:09 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:34:10.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:10 smithi136 ceph-mon[25953]: Deploying daemon osd.5 on smithi136 2024-09-17T09:34:11.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:10 smithi078 ceph-mon[23689]: Deploying daemon osd.5 on smithi136 2024-09-17T09:34:12.056 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:11 smithi136 ceph-mon[25953]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T09:34:12.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:11 smithi078 ceph-mon[23689]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T09:34:13.896 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:13 smithi136 ceph-mon[25953]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T09:34:14.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:13 smithi078 ceph-mon[23689]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T09:34:15.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:15 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:15.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:15 smithi078 ceph-mon[23689]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T09:34:15.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:15 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:34:15.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:15 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:34:15.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:15 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:34:15.848 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:15 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:15.849 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:15 smithi136 ceph-mon[25953]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T09:34:15.849 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:15 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:34:15.849 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:15 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:34:15.849 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:15 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:34:17.632 INFO:teuthology.orchestra.run.smithi136.stdout:Created osd(s) 5 on host 'smithi136' 2024-09-17T09:34:17.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:17 smithi078 ceph-mon[23689]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T09:34:17.938 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:17 smithi136 ceph-mon[25953]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T09:34:18.236 DEBUG:teuthology.orchestra.run.smithi136:osd.5> sudo journalctl -f -n 0 -u ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.5.service 2024-09-17T09:34:18.239 INFO:tasks.cephadm:Deploying osd.6 on smithi136 with /dev/vg_nvme/lv_2... 2024-09-17T09:34:18.239 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-17T09:34:18.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:18 smithi078 ceph-mon[23689]: from='osd.5 [v2:172.21.15.136:6808/2956018891,v1:172.21.15.136:6809/2956018891]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T09:34:18.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:18 smithi078 ceph-mon[23689]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T09:34:18.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:18 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:18.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:18 smithi136 ceph-mon[25953]: from='osd.5 [v2:172.21.15.136:6808/2956018891,v1:172.21.15.136:6809/2956018891]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T09:34:18.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:18 smithi136 ceph-mon[25953]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T09:34:18.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:18 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:19.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:19 smithi136 ceph-mon[25953]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T09:34:19.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:19 smithi136 ceph-mon[25953]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-17T09:34:19.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:19 smithi136 ceph-mon[25953]: osdmap e38: 6 total, 5 up, 6 in 2024-09-17T09:34:19.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:19 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T09:34:19.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:19 smithi136 ceph-mon[25953]: from='osd.5 [v2:172.21.15.136:6808/2956018891,v1:172.21.15.136:6809/2956018891]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-17T09:34:19.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:19 smithi136 ceph-mon[25953]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-17T09:34:19.726 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:34:19 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[33146]: 2024-09-17T09:34:19.560+0000 7f1c81575700 -1 osd.5 0 waiting for initial osdmap 2024-09-17T09:34:19.726 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:34:19 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[33146]: 2024-09-17T09:34:19.567+0000 7f1c7c70d700 -1 osd.5 39 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T09:34:19.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:19 smithi078 ceph-mon[23689]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T09:34:19.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:19 smithi078 ceph-mon[23689]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-17T09:34:19.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:19 smithi078 ceph-mon[23689]: osdmap e38: 6 total, 5 up, 6 in 2024-09-17T09:34:19.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:19 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T09:34:19.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:19 smithi078 ceph-mon[23689]: from='osd.5 [v2:172.21.15.136:6808/2956018891,v1:172.21.15.136:6809/2956018891]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-17T09:34:19.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:19 smithi078 ceph-mon[23689]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-17T09:34:20.842 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:20 smithi136 ceph-mon[25953]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2024-09-17T09:34:20.843 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:20 smithi136 ceph-mon[25953]: osdmap e39: 6 total, 5 up, 6 in 2024-09-17T09:34:20.843 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:20 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T09:34:20.843 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:20 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T09:34:20.843 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:20 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:20.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:20 smithi078 ceph-mon[23689]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2024-09-17T09:34:20.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:20 smithi078 ceph-mon[23689]: osdmap e39: 6 total, 5 up, 6 in 2024-09-17T09:34:20.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:20 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T09:34:20.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:20 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T09:34:20.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:20 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:21.765 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-17T09:34:21.785 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi136:vg_nvme/lv_2 2024-09-17T09:34:21.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:21 smithi078 ceph-mon[23689]: purged_snaps scrub starts 2024-09-17T09:34:21.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:21 smithi078 ceph-mon[23689]: purged_snaps scrub ok 2024-09-17T09:34:21.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:21 smithi078 ceph-mon[23689]: pgmap v130: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T09:34:21.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:21 smithi078 ceph-mon[23689]: osd.5 [v2:172.21.15.136:6808/2956018891,v1:172.21.15.136:6809/2956018891] boot 2024-09-17T09:34:21.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:21 smithi078 ceph-mon[23689]: osdmap e40: 6 total, 6 up, 6 in 2024-09-17T09:34:21.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:21 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T09:34:21.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:21 smithi078 ceph-mon[23689]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-17T09:34:21.946 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:21 smithi136 ceph-mon[25953]: purged_snaps scrub starts 2024-09-17T09:34:21.946 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:21 smithi136 ceph-mon[25953]: purged_snaps scrub ok 2024-09-17T09:34:21.946 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:21 smithi136 ceph-mon[25953]: pgmap v130: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T09:34:21.946 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:21 smithi136 ceph-mon[25953]: osd.5 [v2:172.21.15.136:6808/2956018891,v1:172.21.15.136:6809/2956018891] boot 2024-09-17T09:34:21.946 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:21 smithi136 ceph-mon[25953]: osdmap e40: 6 total, 6 up, 6 in 2024-09-17T09:34:21.946 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:21 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T09:34:21.946 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:21 smithi136 ceph-mon[25953]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-17T09:34:22.801 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:22 smithi136 ceph-mon[25953]: osdmap e41: 6 total, 6 up, 6 in 2024-09-17T09:34:22.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:22 smithi078 ceph-mon[23689]: osdmap e41: 6 total, 6 up, 6 in 2024-09-17T09:34:23.826 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:23 smithi136 ceph-mon[25953]: pgmap v133: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-17T09:34:23.826 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:23 smithi136 ceph-mon[25953]: osdmap e42: 6 total, 6 up, 6 in 2024-09-17T09:34:23.826 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:23 smithi136 ceph-mon[25953]: osdmap e43: 6 total, 6 up, 6 in 2024-09-17T09:34:23.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:23 smithi078 ceph-mon[23689]: pgmap v133: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-17T09:34:23.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:23 smithi078 ceph-mon[23689]: osdmap e42: 6 total, 6 up, 6 in 2024-09-17T09:34:23.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:23 smithi078 ceph-mon[23689]: osdmap e43: 6 total, 6 up, 6 in 2024-09-17T09:34:25.778 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:25 smithi136 ceph-mon[25953]: pgmap v136: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-17T09:34:25.778 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:25 smithi136 ceph-mon[25953]: osdmap e44: 6 total, 6 up, 6 in 2024-09-17T09:34:25.778 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:25 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:25.778 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:25 smithi136 ceph-mon[25953]: from='client.24177 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:34:25.778 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:25 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T09:34:25.779 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:25 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T09:34:25.779 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:25 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:34:25.779 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:25 smithi136 ceph-mon[25953]: Detected new or changed devices on smithi136 2024-09-17T09:34:25.779 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:25 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:25.779 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:25 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:34:25.779 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:25 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:34:25.779 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:25 smithi136 ceph-mon[25953]: Adjusting osd_memory_target on smithi136 to 7968M 2024-09-17T09:34:25.779 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:25 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:25.779 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:25 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:25.779 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:25 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:34:25.779 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:25 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:34:25.779 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:25 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:34:25.779 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:25 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:25.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:25 smithi078 ceph-mon[23689]: pgmap v136: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-17T09:34:25.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:25 smithi078 ceph-mon[23689]: osdmap e44: 6 total, 6 up, 6 in 2024-09-17T09:34:25.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:25 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:25.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:25 smithi078 ceph-mon[23689]: from='client.24177 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:34:25.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:25 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T09:34:25.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:25 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T09:34:25.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:25 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:34:25.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:25 smithi078 ceph-mon[23689]: Detected new or changed devices on smithi136 2024-09-17T09:34:25.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:25 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:25.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:25 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:34:25.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:25 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:34:25.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:25 smithi078 ceph-mon[23689]: Adjusting osd_memory_target on smithi136 to 7968M 2024-09-17T09:34:25.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:25 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:25.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:25 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:25.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:25 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:34:25.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:25 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:34:25.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:25 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:34:25.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:25 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:27.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:27 smithi078 ceph-mon[23689]: pgmap v138: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-17T09:34:27.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:27 smithi078 ceph-mon[23689]: from='client.? 172.21.15.136:0/2029194348' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1e7c585f-b7eb-46a5-8109-2534928f990f"}]: dispatch 2024-09-17T09:34:27.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:27 smithi078 ceph-mon[23689]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1e7c585f-b7eb-46a5-8109-2534928f990f"}]: dispatch 2024-09-17T09:34:27.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:27 smithi078 ceph-mon[23689]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1e7c585f-b7eb-46a5-8109-2534928f990f"}]': finished 2024-09-17T09:34:27.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:27 smithi078 ceph-mon[23689]: osdmap e45: 7 total, 6 up, 7 in 2024-09-17T09:34:27.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:27 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T09:34:27.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:27 smithi136 ceph-mon[25953]: pgmap v138: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-17T09:34:27.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:27 smithi136 ceph-mon[25953]: from='client.? 172.21.15.136:0/2029194348' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1e7c585f-b7eb-46a5-8109-2534928f990f"}]: dispatch 2024-09-17T09:34:27.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:27 smithi136 ceph-mon[25953]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1e7c585f-b7eb-46a5-8109-2534928f990f"}]: dispatch 2024-09-17T09:34:27.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:27 smithi136 ceph-mon[25953]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1e7c585f-b7eb-46a5-8109-2534928f990f"}]': finished 2024-09-17T09:34:27.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:27 smithi136 ceph-mon[25953]: osdmap e45: 7 total, 6 up, 7 in 2024-09-17T09:34:27.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:27 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T09:34:28.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:28 smithi078 ceph-mon[23689]: from='client.? 172.21.15.136:0/1267706936' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T09:34:28.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:28 smithi136 ceph-mon[25953]: from='client.? 172.21.15.136:0/1267706936' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T09:34:29.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:29 smithi078 ceph-mon[23689]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-09-17T09:34:29.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:29 smithi078 ceph-mon[23689]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-17T09:34:29.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:29 smithi078 ceph-mon[23689]: Cluster is now healthy 2024-09-17T09:34:29.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:29 smithi136 ceph-mon[25953]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-09-17T09:34:29.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:29 smithi136 ceph-mon[25953]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-17T09:34:29.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:29 smithi136 ceph-mon[25953]: Cluster is now healthy 2024-09-17T09:34:31.652 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:31 smithi136 ceph-mon[25953]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-09-17T09:34:31.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:31 smithi078 ceph-mon[23689]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-09-17T09:34:33.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:33 smithi136 ceph-mon[25953]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-17T09:34:33.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:33 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T09:34:33.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:33 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:34:33.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:33 smithi078 ceph-mon[23689]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-17T09:34:33.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:33 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T09:34:33.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:33 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:34:34.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:34 smithi078 ceph-mon[23689]: Deploying daemon osd.6 on smithi136 2024-09-17T09:34:34.892 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:34 smithi136 ceph-mon[25953]: Deploying daemon osd.6 on smithi136 2024-09-17T09:34:35.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:35 smithi078 ceph-mon[23689]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T09:34:35.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:35 smithi136 ceph-mon[25953]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T09:34:37.655 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:37 smithi136 ceph-mon[25953]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T09:34:37.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:37 smithi078 ceph-mon[23689]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T09:34:39.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:39 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:39.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:39 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:34:39.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:39 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:34:39.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:39 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:34:39.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:39 smithi136 ceph-mon[25953]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T09:34:39.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:39 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:39.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:39 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:34:39.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:39 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:34:39.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:39 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:34:39.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:39 smithi078 ceph-mon[23689]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T09:34:40.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:40 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:40.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:40 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:41.348 INFO:teuthology.orchestra.run.smithi136.stdout:Created osd(s) 6 on host 'smithi136' 2024-09-17T09:34:41.690 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:41 smithi136 ceph-mon[25953]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T09:34:41.690 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:41 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:41.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:41 smithi078 ceph-mon[23689]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T09:34:41.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:41 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:42.304 DEBUG:teuthology.orchestra.run.smithi136:osd.6> sudo journalctl -f -n 0 -u ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.6.service 2024-09-17T09:34:42.307 INFO:tasks.cephadm:Deploying osd.7 on smithi136 with /dev/vg_nvme/lv_1... 2024-09-17T09:34:42.308 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-17T09:34:42.634 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:42 smithi136 ceph-mon[25953]: from='osd.6 [v2:172.21.15.136:6816/1716373842,v1:172.21.15.136:6817/1716373842]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T09:34:42.634 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:42 smithi136 ceph-mon[25953]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T09:34:42.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:42 smithi078 ceph-mon[23689]: from='osd.6 [v2:172.21.15.136:6816/1716373842,v1:172.21.15.136:6817/1716373842]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T09:34:42.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:42 smithi078 ceph-mon[23689]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T09:34:43.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:43 smithi136 ceph-mon[25953]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T09:34:43.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:43 smithi136 ceph-mon[25953]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-17T09:34:43.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:43 smithi136 ceph-mon[25953]: osdmap e46: 7 total, 6 up, 7 in 2024-09-17T09:34:43.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:43 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T09:34:43.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:43 smithi136 ceph-mon[25953]: from='osd.6 [v2:172.21.15.136:6816/1716373842,v1:172.21.15.136:6817/1716373842]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-17T09:34:43.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:43 smithi136 ceph-mon[25953]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-17T09:34:43.726 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:34:43 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[36113]: 2024-09-17T09:34:43.529+0000 7f465d230700 -1 osd.6 0 waiting for initial osdmap 2024-09-17T09:34:43.726 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:34:43 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[36113]: 2024-09-17T09:34:43.537+0000 7f46583c8700 -1 osd.6 47 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T09:34:43.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:43 smithi078 ceph-mon[23689]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T09:34:43.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:43 smithi078 ceph-mon[23689]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-17T09:34:43.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:43 smithi078 ceph-mon[23689]: osdmap e46: 7 total, 6 up, 7 in 2024-09-17T09:34:43.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:43 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T09:34:43.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:43 smithi078 ceph-mon[23689]: from='osd.6 [v2:172.21.15.136:6816/1716373842,v1:172.21.15.136:6817/1716373842]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-17T09:34:43.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:43 smithi078 ceph-mon[23689]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-17T09:34:44.627 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:44 smithi136 ceph-mon[25953]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2024-09-17T09:34:44.627 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:44 smithi136 ceph-mon[25953]: osdmap e47: 7 total, 6 up, 7 in 2024-09-17T09:34:44.627 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:44 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T09:34:44.627 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:44 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T09:34:44.627 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:44 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:44.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:44 smithi078 ceph-mon[23689]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2024-09-17T09:34:44.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:44 smithi078 ceph-mon[23689]: osdmap e47: 7 total, 6 up, 7 in 2024-09-17T09:34:44.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:44 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T09:34:44.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:44 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T09:34:44.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:44 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:45.646 INFO:teuthology.orchestra.run.smithi136.stdout: 2024-09-17T09:34:45.664 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi136:vg_nvme/lv_1 2024-09-17T09:34:45.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:45 smithi078 ceph-mon[23689]: purged_snaps scrub starts 2024-09-17T09:34:45.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:45 smithi078 ceph-mon[23689]: purged_snaps scrub ok 2024-09-17T09:34:45.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:45 smithi078 ceph-mon[23689]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T09:34:45.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:45 smithi078 ceph-mon[23689]: osd.6 [v2:172.21.15.136:6816/1716373842,v1:172.21.15.136:6817/1716373842] boot 2024-09-17T09:34:45.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:45 smithi078 ceph-mon[23689]: osdmap e48: 7 total, 7 up, 7 in 2024-09-17T09:34:45.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:45 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T09:34:45.853 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:45 smithi136 ceph-mon[25953]: purged_snaps scrub starts 2024-09-17T09:34:45.853 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:45 smithi136 ceph-mon[25953]: purged_snaps scrub ok 2024-09-17T09:34:45.853 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:45 smithi136 ceph-mon[25953]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T09:34:45.853 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:45 smithi136 ceph-mon[25953]: osd.6 [v2:172.21.15.136:6816/1716373842,v1:172.21.15.136:6817/1716373842] boot 2024-09-17T09:34:45.853 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:45 smithi136 ceph-mon[25953]: osdmap e48: 7 total, 7 up, 7 in 2024-09-17T09:34:45.853 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:45 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T09:34:46.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:46 smithi078 ceph-mon[23689]: osdmap e49: 7 total, 7 up, 7 in 2024-09-17T09:34:46.898 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:46 smithi136 ceph-mon[25953]: osdmap e49: 7 total, 7 up, 7 in 2024-09-17T09:34:47.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:47 smithi078 ceph-mon[23689]: pgmap v153: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-17T09:34:47.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:47 smithi078 ceph-mon[23689]: osdmap e50: 7 total, 7 up, 7 in 2024-09-17T09:34:47.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:47 smithi136 ceph-mon[25953]: pgmap v153: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-17T09:34:47.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:47 smithi136 ceph-mon[25953]: osdmap e50: 7 total, 7 up, 7 in 2024-09-17T09:34:48.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:48 smithi078 ceph-mon[23689]: osdmap e51: 7 total, 7 up, 7 in 2024-09-17T09:34:48.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:48 smithi078 ceph-mon[23689]: osdmap e52: 7 total, 7 up, 7 in 2024-09-17T09:34:48.869 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:48 smithi136 ceph-mon[25953]: osdmap e51: 7 total, 7 up, 7 in 2024-09-17T09:34:48.869 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:48 smithi136 ceph-mon[25953]: osdmap e52: 7 total, 7 up, 7 in 2024-09-17T09:34:49.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:49 smithi136 ceph-mon[25953]: pgmap v156: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T09:34:49.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:49 smithi136 ceph-mon[25953]: from='client.24195 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:34:49.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T09:34:49.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T09:34:49.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:34:49.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:49 smithi078 ceph-mon[23689]: pgmap v156: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T09:34:49.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:49 smithi078 ceph-mon[23689]: from='client.24195 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:34:49.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T09:34:49.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T09:34:49.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:34:51.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:51 smithi136 ceph-mon[25953]: Detected new or changed devices on smithi136 2024-09-17T09:34:51.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:51 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:51.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:51 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:34:51.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:51 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:34:51.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:51 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:34:51.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:51 smithi136 ceph-mon[25953]: Adjusting osd_memory_target on smithi136 to 5312M 2024-09-17T09:34:51.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:51 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:51.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:51 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:51.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:51 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:34:51.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:51 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:34:51.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:51 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:34:51.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:51 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:51.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:51 smithi136 ceph-mon[25953]: pgmap v158: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T09:34:51.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:51 smithi078 ceph-mon[23689]: Detected new or changed devices on smithi136 2024-09-17T09:34:51.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:51 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:51.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:51 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:34:51.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:51 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:34:51.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:51 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:34:51.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:51 smithi078 ceph-mon[23689]: Adjusting osd_memory_target on smithi136 to 5312M 2024-09-17T09:34:51.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:51 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:51.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:51 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:51.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:51 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:34:51.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:51 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:34:51.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:51 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:34:51.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:51 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:34:51.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:51 smithi078 ceph-mon[23689]: pgmap v158: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T09:34:52.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:52 smithi078 ceph-mon[23689]: from='client.? 172.21.15.136:0/2420829997' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "504bffb3-f118-46e6-ab6c-bb6a807b29ee"}]: dispatch 2024-09-17T09:34:52.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:52 smithi078 ceph-mon[23689]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "504bffb3-f118-46e6-ab6c-bb6a807b29ee"}]: dispatch 2024-09-17T09:34:52.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:52 smithi078 ceph-mon[23689]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "504bffb3-f118-46e6-ab6c-bb6a807b29ee"}]': finished 2024-09-17T09:34:52.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:52 smithi078 ceph-mon[23689]: osdmap e53: 8 total, 7 up, 8 in 2024-09-17T09:34:52.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:52 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T09:34:52.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:52 smithi136 ceph-mon[25953]: from='client.? 172.21.15.136:0/2420829997' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "504bffb3-f118-46e6-ab6c-bb6a807b29ee"}]: dispatch 2024-09-17T09:34:52.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:52 smithi136 ceph-mon[25953]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "504bffb3-f118-46e6-ab6c-bb6a807b29ee"}]: dispatch 2024-09-17T09:34:52.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:52 smithi136 ceph-mon[25953]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "504bffb3-f118-46e6-ab6c-bb6a807b29ee"}]': finished 2024-09-17T09:34:52.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:52 smithi136 ceph-mon[25953]: osdmap e53: 8 total, 7 up, 8 in 2024-09-17T09:34:52.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:52 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T09:34:53.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:53 smithi136 ceph-mon[25953]: from='client.? 172.21.15.136:0/978288309' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T09:34:53.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:53 smithi136 ceph-mon[25953]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-09-17T09:34:53.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:53 smithi078 ceph-mon[23689]: from='client.? 172.21.15.136:0/978288309' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T09:34:53.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:53 smithi078 ceph-mon[23689]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-09-17T09:34:55.834 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:55 smithi136 ceph-mon[25953]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-09-17T09:34:55.834 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:55 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/trash_purge_schedule"}]: dispatch 2024-09-17T09:34:55.834 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:55 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:34:55.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:55 smithi078 ceph-mon[23689]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-09-17T09:34:55.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:55 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/trash_purge_schedule"}]: dispatch 2024-09-17T09:34:55.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:55 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:34:57.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:57 smithi136 ceph-mon[25953]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-17T09:34:57.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:57 smithi078 ceph-mon[23689]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-17T09:34:58.707 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:58 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T09:34:58.707 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:58 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:34:58.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:58 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T09:34:58.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:58 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:34:59.805 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:59 smithi136 ceph-mon[25953]: Deploying daemon osd.7 on smithi136 2024-09-17T09:34:59.805 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:34:59 smithi136 ceph-mon[25953]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T09:34:59.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:59 smithi078 ceph-mon[23689]: Deploying daemon osd.7 on smithi136 2024-09-17T09:34:59.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:34:59 smithi078 ceph-mon[23689]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T09:35:01.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:01 smithi078 ceph-mon[23689]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T09:35:01.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:01 smithi136 ceph-mon[25953]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T09:35:03.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:03 smithi078 ceph-mon[23689]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-17T09:35:03.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:03 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:35:03.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:03 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:35:03.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:03 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:35:03.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:03 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:35:03.654 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:03 smithi136 ceph-mon[25953]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-17T09:35:03.654 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:03 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:35:03.654 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:03 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:35:03.654 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:03 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:35:03.654 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:03 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:35:05.777 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:05 smithi136 ceph-mon[25953]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-17T09:35:05.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:05 smithi078 ceph-mon[23689]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-17T09:35:06.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:06 smithi136 ceph-mon[25953]: from='osd.7 [v2:172.21.15.136:6824/3497474558,v1:172.21.15.136:6825/3497474558]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T09:35:06.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:06 smithi136 ceph-mon[25953]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T09:35:06.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:06 smithi078 ceph-mon[23689]: from='osd.7 [v2:172.21.15.136:6824/3497474558,v1:172.21.15.136:6825/3497474558]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T09:35:06.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:06 smithi078 ceph-mon[23689]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T09:35:06.873 INFO:teuthology.orchestra.run.smithi136.stdout:Created osd(s) 7 on host 'smithi136' 2024-09-17T09:35:07.521 DEBUG:teuthology.orchestra.run.smithi136:osd.7> sudo journalctl -f -n 0 -u ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.7.service 2024-09-17T09:35:07.524 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-17T09:35:07.524 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-17T09:35:07.830 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:07 smithi078 ceph-mon[23689]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-17T09:35:07.830 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:07 smithi078 ceph-mon[23689]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-17T09:35:07.830 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:07 smithi078 ceph-mon[23689]: osdmap e54: 8 total, 7 up, 8 in 2024-09-17T09:35:07.831 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:07 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T09:35:07.831 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:07 smithi078 ceph-mon[23689]: from='osd.7 [v2:172.21.15.136:6824/3497474558,v1:172.21.15.136:6825/3497474558]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-17T09:35:07.831 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:07 smithi078 ceph-mon[23689]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-17T09:35:07.831 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:07 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:35:07.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:07 smithi136 ceph-mon[25953]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-17T09:35:07.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:07 smithi136 ceph-mon[25953]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-17T09:35:07.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:07 smithi136 ceph-mon[25953]: osdmap e54: 8 total, 7 up, 8 in 2024-09-17T09:35:07.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:07 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T09:35:07.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:07 smithi136 ceph-mon[25953]: from='osd.7 [v2:172.21.15.136:6824/3497474558,v1:172.21.15.136:6825/3497474558]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-17T09:35:07.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:07 smithi136 ceph-mon[25953]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2024-09-17T09:35:07.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:07 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:35:08.700 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:08 smithi136 ceph-mon[25953]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2024-09-17T09:35:08.701 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:08 smithi136 ceph-mon[25953]: osdmap e55: 8 total, 7 up, 8 in 2024-09-17T09:35:08.701 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:08 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T09:35:08.701 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:08 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T09:35:08.701 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:08 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:35:08.701 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:08 smithi136 ceph-mon[25953]: osd.7 [v2:172.21.15.136:6824/3497474558,v1:172.21.15.136:6825/3497474558] boot 2024-09-17T09:35:08.701 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:08 smithi136 ceph-mon[25953]: osdmap e56: 8 total, 8 up, 8 in 2024-09-17T09:35:08.701 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:08 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T09:35:08.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:08 smithi078 ceph-mon[23689]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2024-09-17T09:35:08.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:08 smithi078 ceph-mon[23689]: osdmap e55: 8 total, 7 up, 8 in 2024-09-17T09:35:08.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:08 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T09:35:08.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:08 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T09:35:08.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:08 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:35:08.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:08 smithi078 ceph-mon[23689]: osd.7 [v2:172.21.15.136:6824/3497474558,v1:172.21.15.136:6825/3497474558] boot 2024-09-17T09:35:08.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:08 smithi078 ceph-mon[23689]: osdmap e56: 8 total, 8 up, 8 in 2024-09-17T09:35:08.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:08 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T09:35:09.601 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-17T09:35:09.616 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:09 smithi136 ceph-mon[25953]: purged_snaps scrub starts 2024-09-17T09:35:09.616 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:09 smithi136 ceph-mon[25953]: purged_snaps scrub ok 2024-09-17T09:35:09.617 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:09 smithi136 ceph-mon[25953]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-17T09:35:09.617 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:09 smithi136 ceph-mon[25953]: osdmap e57: 8 total, 8 up, 8 in 2024-09-17T09:35:09.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:09 smithi078 ceph-mon[23689]: purged_snaps scrub starts 2024-09-17T09:35:09.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:09 smithi078 ceph-mon[23689]: purged_snaps scrub ok 2024-09-17T09:35:09.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:09 smithi078 ceph-mon[23689]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-17T09:35:09.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:09 smithi078 ceph-mon[23689]: osdmap e57: 8 total, 8 up, 8 in 2024-09-17T09:35:10.212 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":57,"num_osds":8,"num_up_osds":8,"osd_up_since":1726565708,"num_in_osds":8,"osd_in_since":1726565691,"num_remapped_pgs":1} 2024-09-17T09:35:10.212 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-17T09:35:10.426 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-17T09:35:10.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:10 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/696490678' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-17T09:35:10.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:10 smithi136 ceph-mon[25953]: osdmap e58: 8 total, 8 up, 8 in 2024-09-17T09:35:10.791 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:10 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/696490678' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-17T09:35:10.791 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:10 smithi078 ceph-mon[23689]: osdmap e58: 8 total, 8 up, 8 in 2024-09-17T09:35:11.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:11 smithi078 ceph-mon[23689]: pgmap v173: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:11.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:11 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:35:11.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:11 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:35:11.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:11 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:35:11.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:11 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:35:11.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:11 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:35:11.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:11 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:35:11.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:11 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:35:11.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:11 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:35:11.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:11 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:35:11.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:11 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:35:11.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:11 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:35:11.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:11 smithi136 ceph-mon[25953]: pgmap v173: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:11.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:11 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:35:11.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:11 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:35:11.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:11 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:35:11.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:11 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:35:11.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:11 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:35:11.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:11 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:35:11.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:11 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:35:11.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:11 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:35:11.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:11 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:35:11.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:11 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:35:11.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:11 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:35:12.308 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-17T09:35:12.309 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":59,"fsid":"35c0cd22-74d7-11ef-bceb-c7b262605968","created":"2024-09-17T09:29:04.647474+0000","modified":"2024-09-17T09:35:11.541156+0000","last_up_change":"2024-09-17T09:35:08.507108+0000","last_in_change":"2024-09-17T09:34:51.792525+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-17T09:33:08.775342+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"6666cf1b-b4e3-48d3-a076-be328ec75fb6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":2296669224},{"type":"v1","addr":"172.21.15.78:6803","nonce":2296669224}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":2296669224},{"type":"v1","addr":"172.21.15.78:6805","nonce":2296669224}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":2296669224},{"type":"v1","addr":"172.21.15.78:6809","nonce":2296669224}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":2296669224},{"type":"v1","addr":"172.21.15.78:6807","nonce":2296669224}]},"public_addr":"172.21.15.78:6803/2296669224","cluster_addr":"172.21.15.78:6805/2296669224","heartbeat_back_addr":"172.21.15.78:6809/2296669224","heartbeat_front_addr":"172.21.15.78:6807/2296669224","state":["exists","up"]},{"osd":1,"uuid":"5d895d59-82d7-40da-9591-42d8f9b258b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":198469681},{"type":"v1","addr":"172.21.15.78:6811","nonce":198469681}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":198469681},{"type":"v1","addr":"172.21.15.78:6813","nonce":198469681}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":198469681},{"type":"v1","addr":"172.21.15.78:6817","nonce":198469681}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":198469681},{"type":"v1","addr":"172.21.15.78:6815","nonce":198469681}]},"public_addr":"172.21.15.78:6811/198469681","cluster_addr":"172.21.15.78:6813/198469681","heartbeat_back_addr":"172.21.15.78:6817/198469681","heartbeat_front_addr":"172.21.15.78:6815/198469681","state":["exists","up"]},{"osd":2,"uuid":"fe89957a-ef6e-4208-9fe8-6f1078511f26","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":1349073215},{"type":"v1","addr":"172.21.15.78:6819","nonce":1349073215}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":1349073215},{"type":"v1","addr":"172.21.15.78:6821","nonce":1349073215}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6824","nonce":1349073215},{"type":"v1","addr":"172.21.15.78:6825","nonce":1349073215}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":1349073215},{"type":"v1","addr":"172.21.15.78:6823","nonce":1349073215}]},"public_addr":"172.21.15.78:6819/1349073215","cluster_addr":"172.21.15.78:6821/1349073215","heartbeat_back_addr":"172.21.15.78:6825/1349073215","heartbeat_front_addr":"172.21.15.78:6823/1349073215","state":["exists","up"]},{"osd":3,"uuid":"b4106928-6750-4445-bef2-d5c216e9ae7c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6826","nonce":3175336990},{"type":"v1","addr":"172.21.15.78:6827","nonce":3175336990}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6828","nonce":3175336990},{"type":"v1","addr":"172.21.15.78:6829","nonce":3175336990}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6832","nonce":3175336990},{"type":"v1","addr":"172.21.15.78:6833","nonce":3175336990}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6830","nonce":3175336990},{"type":"v1","addr":"172.21.15.78:6831","nonce":3175336990}]},"public_addr":"172.21.15.78:6827/3175336990","cluster_addr":"172.21.15.78:6829/3175336990","heartbeat_back_addr":"172.21.15.78:6833/3175336990","heartbeat_front_addr":"172.21.15.78:6831/3175336990","state":["exists","up"]},{"osd":4,"uuid":"3334c021-ced7-4aff-9ee4-77e6ef91e87c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6800","nonce":2032814238},{"type":"v1","addr":"172.21.15.136:6801","nonce":2032814238}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6802","nonce":2032814238},{"type":"v1","addr":"172.21.15.136:6803","nonce":2032814238}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6806","nonce":2032814238},{"type":"v1","addr":"172.21.15.136:6807","nonce":2032814238}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6804","nonce":2032814238},{"type":"v1","addr":"172.21.15.136:6805","nonce":2032814238}]},"public_addr":"172.21.15.136:6801/2032814238","cluster_addr":"172.21.15.136:6803/2032814238","heartbeat_back_addr":"172.21.15.136:6807/2032814238","heartbeat_front_addr":"172.21.15.136:6805/2032814238","state":["exists","up"]},{"osd":5,"uuid":"3b900ce7-186b-45dc-a15c-5b078321c9f4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6808","nonce":2956018891},{"type":"v1","addr":"172.21.15.136:6809","nonce":2956018891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6810","nonce":2956018891},{"type":"v1","addr":"172.21.15.136:6811","nonce":2956018891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6814","nonce":2956018891},{"type":"v1","addr":"172.21.15.136:6815","nonce":2956018891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6812","nonce":2956018891},{"type":"v1","addr":"172.21.15.136:6813","nonce":2956018891}]},"public_addr":"172.21.15.136:6809/2956018891","cluster_addr":"172.21.15.136:6811/2956018891","heartbeat_back_addr":"172.21.15.136:6815/2956018891","heartbeat_front_addr":"172.21.15.136:6813/2956018891","state":["exists","up"]},{"osd":6,"uuid":"1e7c585f-b7eb-46a5-8109-2534928f990f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6816","nonce":1716373842},{"type":"v1","addr":"172.21.15.136:6817","nonce":1716373842}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6818","nonce":1716373842},{"type":"v1","addr":"172.21.15.136:6819","nonce":1716373842}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6822","nonce":1716373842},{"type":"v1","addr":"172.21.15.136:6823","nonce":1716373842}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6820","nonce":1716373842},{"type":"v1","addr":"172.21.15.136:6821","nonce":1716373842}]},"public_addr":"172.21.15.136:6817/1716373842","cluster_addr":"172.21.15.136:6819/1716373842","heartbeat_back_addr":"172.21.15.136:6823/1716373842","heartbeat_front_addr":"172.21.15.136:6821/1716373842","state":["exists","up"]},{"osd":7,"uuid":"504bffb3-f118-46e6-ab6c-bb6a807b29ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6824","nonce":3497474558},{"type":"v1","addr":"172.21.15.136:6825","nonce":3497474558}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6826","nonce":3497474558},{"type":"v1","addr":"172.21.15.136:6827","nonce":3497474558}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6830","nonce":3497474558},{"type":"v1","addr":"172.21.15.136:6831","nonce":3497474558}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6828","nonce":3497474558},{"type":"v1","addr":"172.21.15.136:6829","nonce":3497474558}]},"public_addr":"172.21.15.136:6825/3497474558","cluster_addr":"172.21.15.136:6827/3497474558","heartbeat_back_addr":"172.21.15.136:6831/3497474558","heartbeat_front_addr":"172.21.15.136:6829/3497474558","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T09:32:07.432854+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T09:32:39.270927+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T09:33:05.297405+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T09:33:31.722790+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T09:33:53.700594+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T09:34:18.634718+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T09:34:42.572076+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T09:35:07.388966+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.78:6801/3049272511":"2024-09-18T09:30:53.854987+0000","172.21.15.78:6800/3049272511":"2024-09-18T09:30:53.854987+0000","172.21.15.78:0/2217269551":"2024-09-18T09:30:53.854987+0000","172.21.15.78:0/3448812961":"2024-09-18T09:30:04.243833+0000","172.21.15.78:0/2050832447":"2024-09-18T09:30:04.243833+0000","172.21.15.78:0/2201892685":"2024-09-18T09:30:53.854987+0000","172.21.15.78:6801/3053679137":"2024-09-18T09:30:04.243833+0000","172.21.15.78:6800/3053679137":"2024-09-18T09:30:04.243833+0000","172.21.15.78:0/1896225568":"2024-09-18T09:30:53.854987+0000","172.21.15.78:0/1100384719":"2024-09-18T09:30:04.243833+0000","172.21.15.78:0/2571571226":"2024-09-18T09:29:33.964265+0000","172.21.15.78:6800/2661840943":"2024-09-18T09:29:33.964265+0000","172.21.15.78:6801/2661840943":"2024-09-18T09:29:33.964265+0000","172.21.15.78:0/2015147337":"2024-09-18T09:29:33.964265+0000","172.21.15.78:0/2030020465":"2024-09-18T09:29:33.964265+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-17T09:35:12.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:12 smithi078 ceph-mon[23689]: Detected new or changed devices on smithi136 2024-09-17T09:35:12.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:12 smithi078 ceph-mon[23689]: Adjusting osd_memory_target on smithi136 to 3984M 2024-09-17T09:35:12.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:12 smithi078 ceph-mon[23689]: osdmap e59: 8 total, 8 up, 8 in 2024-09-17T09:35:12.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:12 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/2349245279' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T09:35:12.892 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-17T09:33:08.775342+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '23', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-17T09:35:12.892 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-09-17T09:35:12.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:12 smithi136 ceph-mon[25953]: Detected new or changed devices on smithi136 2024-09-17T09:35:12.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:12 smithi136 ceph-mon[25953]: Adjusting osd_memory_target on smithi136 to 3984M 2024-09-17T09:35:12.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:12 smithi136 ceph-mon[25953]: osdmap e59: 8 total, 8 up, 8 in 2024-09-17T09:35:12.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:12 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/2349245279' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T09:35:13.100 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-17T09:35:13.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:13 smithi078 ceph-mon[23689]: pgmap v176: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:13.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:13 smithi078 ceph-mon[23689]: osdmap e60: 8 total, 8 up, 8 in 2024-09-17T09:35:13.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:13 smithi136 ceph-mon[25953]: pgmap v176: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:13.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:13 smithi136 ceph-mon[25953]: osdmap e60: 8 total, 8 up, 8 in 2024-09-17T09:35:14.941 INFO:teuthology.orchestra.run.smithi078.stdout:pg_num: 1 2024-09-17T09:35:15.649 INFO:tasks.cephadm:Setting up client nodes... 2024-09-17T09:35:15.649 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-17T09:35:15.673 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:15 smithi078 ceph-mon[23689]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 104 KiB/s, 0 objects/s recovering 2024-09-17T09:35:15.673 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:15 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/1122078179' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-17T09:35:15.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:15 smithi136 ceph-mon[25953]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 104 KiB/s, 0 objects/s recovering 2024-09-17T09:35:15.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:15 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/1122078179' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-17T09:35:17.685 INFO:teuthology.orchestra.run.smithi078.stdout:[client.0] 2024-09-17T09:35:17.685 INFO:teuthology.orchestra.run.smithi078.stdout: key = AQBVTelmEVB4KBAAJyETumo2VKNoh0v2JjGn4w== 2024-09-17T09:35:17.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:17 smithi078 ceph-mon[23689]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-17T09:35:17.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:17 smithi136 ceph-mon[25953]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-17T09:35:18.463 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-17T09:35:18.463 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-17T09:35:18.463 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-17T09:35:18.542 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph config log 1 --format=json 2024-09-17T09:35:18.791 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-17T09:35:18.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:18 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/4130165' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-17T09:35:18.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:18 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/4130165' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-17T09:35:18.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:18 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/4130165' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-17T09:35:18.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:18 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/4130165' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-17T09:35:19.953 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:19 smithi078 ceph-mon[23689]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-17T09:35:19.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:19 smithi136 ceph-mon[25953]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-17T09:35:20.646 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-17T09:35:20.933 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:20 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/3993054424' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-17T09:35:20.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:20 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/3993054424' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-17T09:35:21.345 INFO:teuthology.orchestra.run.smithi078.stdout:[{"version":21,"timestamp":"2024-09-17T09:35:11.162443+0000","name":"","changes":[{"name":"osd/host:smithi136/osd_memory_target","previous_value":"5570051140","new_value":"4177538355"}]}] 2024-09-17T09:35:21.346 INFO:tasks.ceph_manager:config epoch is 21 2024-09-17T09:35:21.346 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-17T09:35:21.346 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-17T09:35:21.346 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-17T09:35:21.560 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-17T09:35:21.934 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:21 smithi078 ceph-mon[23689]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-17T09:35:21.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:21 smithi136 ceph-mon[25953]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-17T09:35:23.299 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-17T09:35:23.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:23 smithi136 ceph-mon[25953]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T09:35:23.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:23 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/2709651866' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-17T09:35:23.991 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:23 smithi078 ceph-mon[23689]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T09:35:23.991 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:23 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/2709651866' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-17T09:35:24.015 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":19,"active_gid":14202,"active_name":"smithi078.rsqofi","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6800","nonce":740975757},{"type":"v1","addr":"172.21.15.78:6801","nonce":740975757}]},"active_addr":"172.21.15.78:6801/740975757","active_change":"2024-09-17T09:30:53.855313+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14224,"name":"smithi136.hicmlm","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.78:8443/","prometheus":"http://172.21.15.78:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":910412881}]},{"addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":396604239}]},{"addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":2003596181}]},{"addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":713474324}]}]}} 2024-09-17T09:35:24.019 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-17T09:35:24.019 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-17T09:35:24.019 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-17T09:35:24.223 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-17T09:35:25.973 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-17T09:35:25.973 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":60,"fsid":"35c0cd22-74d7-11ef-bceb-c7b262605968","created":"2024-09-17T09:29:04.647474+0000","modified":"2024-09-17T09:35:12.546276+0000","last_up_change":"2024-09-17T09:35:08.507108+0000","last_in_change":"2024-09-17T09:34:51.792525+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-17T09:33:08.775342+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"6666cf1b-b4e3-48d3-a076-be328ec75fb6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":2296669224},{"type":"v1","addr":"172.21.15.78:6803","nonce":2296669224}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":2296669224},{"type":"v1","addr":"172.21.15.78:6805","nonce":2296669224}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":2296669224},{"type":"v1","addr":"172.21.15.78:6809","nonce":2296669224}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":2296669224},{"type":"v1","addr":"172.21.15.78:6807","nonce":2296669224}]},"public_addr":"172.21.15.78:6803/2296669224","cluster_addr":"172.21.15.78:6805/2296669224","heartbeat_back_addr":"172.21.15.78:6809/2296669224","heartbeat_front_addr":"172.21.15.78:6807/2296669224","state":["exists","up"]},{"osd":1,"uuid":"5d895d59-82d7-40da-9591-42d8f9b258b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":198469681},{"type":"v1","addr":"172.21.15.78:6811","nonce":198469681}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":198469681},{"type":"v1","addr":"172.21.15.78:6813","nonce":198469681}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":198469681},{"type":"v1","addr":"172.21.15.78:6817","nonce":198469681}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":198469681},{"type":"v1","addr":"172.21.15.78:6815","nonce":198469681}]},"public_addr":"172.21.15.78:6811/198469681","cluster_addr":"172.21.15.78:6813/198469681","heartbeat_back_addr":"172.21.15.78:6817/198469681","heartbeat_front_addr":"172.21.15.78:6815/198469681","state":["exists","up"]},{"osd":2,"uuid":"fe89957a-ef6e-4208-9fe8-6f1078511f26","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":1349073215},{"type":"v1","addr":"172.21.15.78:6819","nonce":1349073215}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":1349073215},{"type":"v1","addr":"172.21.15.78:6821","nonce":1349073215}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6824","nonce":1349073215},{"type":"v1","addr":"172.21.15.78:6825","nonce":1349073215}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":1349073215},{"type":"v1","addr":"172.21.15.78:6823","nonce":1349073215}]},"public_addr":"172.21.15.78:6819/1349073215","cluster_addr":"172.21.15.78:6821/1349073215","heartbeat_back_addr":"172.21.15.78:6825/1349073215","heartbeat_front_addr":"172.21.15.78:6823/1349073215","state":["exists","up"]},{"osd":3,"uuid":"b4106928-6750-4445-bef2-d5c216e9ae7c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6826","nonce":3175336990},{"type":"v1","addr":"172.21.15.78:6827","nonce":3175336990}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6828","nonce":3175336990},{"type":"v1","addr":"172.21.15.78:6829","nonce":3175336990}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6832","nonce":3175336990},{"type":"v1","addr":"172.21.15.78:6833","nonce":3175336990}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6830","nonce":3175336990},{"type":"v1","addr":"172.21.15.78:6831","nonce":3175336990}]},"public_addr":"172.21.15.78:6827/3175336990","cluster_addr":"172.21.15.78:6829/3175336990","heartbeat_back_addr":"172.21.15.78:6833/3175336990","heartbeat_front_addr":"172.21.15.78:6831/3175336990","state":["exists","up"]},{"osd":4,"uuid":"3334c021-ced7-4aff-9ee4-77e6ef91e87c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6800","nonce":2032814238},{"type":"v1","addr":"172.21.15.136:6801","nonce":2032814238}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6802","nonce":2032814238},{"type":"v1","addr":"172.21.15.136:6803","nonce":2032814238}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6806","nonce":2032814238},{"type":"v1","addr":"172.21.15.136:6807","nonce":2032814238}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6804","nonce":2032814238},{"type":"v1","addr":"172.21.15.136:6805","nonce":2032814238}]},"public_addr":"172.21.15.136:6801/2032814238","cluster_addr":"172.21.15.136:6803/2032814238","heartbeat_back_addr":"172.21.15.136:6807/2032814238","heartbeat_front_addr":"172.21.15.136:6805/2032814238","state":["exists","up"]},{"osd":5,"uuid":"3b900ce7-186b-45dc-a15c-5b078321c9f4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6808","nonce":2956018891},{"type":"v1","addr":"172.21.15.136:6809","nonce":2956018891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6810","nonce":2956018891},{"type":"v1","addr":"172.21.15.136:6811","nonce":2956018891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6814","nonce":2956018891},{"type":"v1","addr":"172.21.15.136:6815","nonce":2956018891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6812","nonce":2956018891},{"type":"v1","addr":"172.21.15.136:6813","nonce":2956018891}]},"public_addr":"172.21.15.136:6809/2956018891","cluster_addr":"172.21.15.136:6811/2956018891","heartbeat_back_addr":"172.21.15.136:6815/2956018891","heartbeat_front_addr":"172.21.15.136:6813/2956018891","state":["exists","up"]},{"osd":6,"uuid":"1e7c585f-b7eb-46a5-8109-2534928f990f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6816","nonce":1716373842},{"type":"v1","addr":"172.21.15.136:6817","nonce":1716373842}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6818","nonce":1716373842},{"type":"v1","addr":"172.21.15.136:6819","nonce":1716373842}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6822","nonce":1716373842},{"type":"v1","addr":"172.21.15.136:6823","nonce":1716373842}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6820","nonce":1716373842},{"type":"v1","addr":"172.21.15.136:6821","nonce":1716373842}]},"public_addr":"172.21.15.136:6817/1716373842","cluster_addr":"172.21.15.136:6819/1716373842","heartbeat_back_addr":"172.21.15.136:6823/1716373842","heartbeat_front_addr":"172.21.15.136:6821/1716373842","state":["exists","up"]},{"osd":7,"uuid":"504bffb3-f118-46e6-ab6c-bb6a807b29ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6824","nonce":3497474558},{"type":"v1","addr":"172.21.15.136:6825","nonce":3497474558}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6826","nonce":3497474558},{"type":"v1","addr":"172.21.15.136:6827","nonce":3497474558}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6830","nonce":3497474558},{"type":"v1","addr":"172.21.15.136:6831","nonce":3497474558}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6828","nonce":3497474558},{"type":"v1","addr":"172.21.15.136:6829","nonce":3497474558}]},"public_addr":"172.21.15.136:6825/3497474558","cluster_addr":"172.21.15.136:6827/3497474558","heartbeat_back_addr":"172.21.15.136:6831/3497474558","heartbeat_front_addr":"172.21.15.136:6829/3497474558","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T09:32:07.432854+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T09:32:39.270927+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T09:33:05.297405+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T09:33:31.722790+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T09:33:53.700594+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T09:34:18.634718+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T09:34:42.572076+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T09:35:07.388966+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.78:6801/3049272511":"2024-09-18T09:30:53.854987+0000","172.21.15.78:6800/3049272511":"2024-09-18T09:30:53.854987+0000","172.21.15.78:0/2217269551":"2024-09-18T09:30:53.854987+0000","172.21.15.78:0/3448812961":"2024-09-18T09:30:04.243833+0000","172.21.15.78:0/2050832447":"2024-09-18T09:30:04.243833+0000","172.21.15.78:0/2201892685":"2024-09-18T09:30:53.854987+0000","172.21.15.78:6801/3053679137":"2024-09-18T09:30:04.243833+0000","172.21.15.78:6800/3053679137":"2024-09-18T09:30:04.243833+0000","172.21.15.78:0/1896225568":"2024-09-18T09:30:53.854987+0000","172.21.15.78:0/1100384719":"2024-09-18T09:30:04.243833+0000","172.21.15.78:0/2571571226":"2024-09-18T09:29:33.964265+0000","172.21.15.78:6800/2661840943":"2024-09-18T09:29:33.964265+0000","172.21.15.78:6801/2661840943":"2024-09-18T09:29:33.964265+0000","172.21.15.78:0/2015147337":"2024-09-18T09:29:33.964265+0000","172.21.15.78:0/2030020465":"2024-09-18T09:29:33.964265+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-17T09:35:25.974 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:25 smithi078 ceph-mon[23689]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-09-17T09:35:25.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:25 smithi136 ceph-mon[25953]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-09-17T09:35:26.730 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-17T09:35:26.731 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-17T09:35:26.940 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-17T09:35:26.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:26 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/794638819' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T09:35:27.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:26 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/794638819' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T09:35:27.710 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:27 smithi078 ceph-mon[23689]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:27.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:27 smithi136 ceph-mon[25953]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:28.632 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-17T09:35:28.632 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":60,"fsid":"35c0cd22-74d7-11ef-bceb-c7b262605968","created":"2024-09-17T09:29:04.647474+0000","modified":"2024-09-17T09:35:12.546276+0000","last_up_change":"2024-09-17T09:35:08.507108+0000","last_in_change":"2024-09-17T09:34:51.792525+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-17T09:33:08.775342+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"6666cf1b-b4e3-48d3-a076-be328ec75fb6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":2296669224},{"type":"v1","addr":"172.21.15.78:6803","nonce":2296669224}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":2296669224},{"type":"v1","addr":"172.21.15.78:6805","nonce":2296669224}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":2296669224},{"type":"v1","addr":"172.21.15.78:6809","nonce":2296669224}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":2296669224},{"type":"v1","addr":"172.21.15.78:6807","nonce":2296669224}]},"public_addr":"172.21.15.78:6803/2296669224","cluster_addr":"172.21.15.78:6805/2296669224","heartbeat_back_addr":"172.21.15.78:6809/2296669224","heartbeat_front_addr":"172.21.15.78:6807/2296669224","state":["exists","up"]},{"osd":1,"uuid":"5d895d59-82d7-40da-9591-42d8f9b258b3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":198469681},{"type":"v1","addr":"172.21.15.78:6811","nonce":198469681}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":198469681},{"type":"v1","addr":"172.21.15.78:6813","nonce":198469681}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":198469681},{"type":"v1","addr":"172.21.15.78:6817","nonce":198469681}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":198469681},{"type":"v1","addr":"172.21.15.78:6815","nonce":198469681}]},"public_addr":"172.21.15.78:6811/198469681","cluster_addr":"172.21.15.78:6813/198469681","heartbeat_back_addr":"172.21.15.78:6817/198469681","heartbeat_front_addr":"172.21.15.78:6815/198469681","state":["exists","up"]},{"osd":2,"uuid":"fe89957a-ef6e-4208-9fe8-6f1078511f26","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":1349073215},{"type":"v1","addr":"172.21.15.78:6819","nonce":1349073215}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":1349073215},{"type":"v1","addr":"172.21.15.78:6821","nonce":1349073215}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6824","nonce":1349073215},{"type":"v1","addr":"172.21.15.78:6825","nonce":1349073215}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":1349073215},{"type":"v1","addr":"172.21.15.78:6823","nonce":1349073215}]},"public_addr":"172.21.15.78:6819/1349073215","cluster_addr":"172.21.15.78:6821/1349073215","heartbeat_back_addr":"172.21.15.78:6825/1349073215","heartbeat_front_addr":"172.21.15.78:6823/1349073215","state":["exists","up"]},{"osd":3,"uuid":"b4106928-6750-4445-bef2-d5c216e9ae7c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6826","nonce":3175336990},{"type":"v1","addr":"172.21.15.78:6827","nonce":3175336990}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6828","nonce":3175336990},{"type":"v1","addr":"172.21.15.78:6829","nonce":3175336990}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6832","nonce":3175336990},{"type":"v1","addr":"172.21.15.78:6833","nonce":3175336990}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6830","nonce":3175336990},{"type":"v1","addr":"172.21.15.78:6831","nonce":3175336990}]},"public_addr":"172.21.15.78:6827/3175336990","cluster_addr":"172.21.15.78:6829/3175336990","heartbeat_back_addr":"172.21.15.78:6833/3175336990","heartbeat_front_addr":"172.21.15.78:6831/3175336990","state":["exists","up"]},{"osd":4,"uuid":"3334c021-ced7-4aff-9ee4-77e6ef91e87c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6800","nonce":2032814238},{"type":"v1","addr":"172.21.15.136:6801","nonce":2032814238}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6802","nonce":2032814238},{"type":"v1","addr":"172.21.15.136:6803","nonce":2032814238}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6806","nonce":2032814238},{"type":"v1","addr":"172.21.15.136:6807","nonce":2032814238}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6804","nonce":2032814238},{"type":"v1","addr":"172.21.15.136:6805","nonce":2032814238}]},"public_addr":"172.21.15.136:6801/2032814238","cluster_addr":"172.21.15.136:6803/2032814238","heartbeat_back_addr":"172.21.15.136:6807/2032814238","heartbeat_front_addr":"172.21.15.136:6805/2032814238","state":["exists","up"]},{"osd":5,"uuid":"3b900ce7-186b-45dc-a15c-5b078321c9f4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6808","nonce":2956018891},{"type":"v1","addr":"172.21.15.136:6809","nonce":2956018891}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6810","nonce":2956018891},{"type":"v1","addr":"172.21.15.136:6811","nonce":2956018891}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6814","nonce":2956018891},{"type":"v1","addr":"172.21.15.136:6815","nonce":2956018891}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6812","nonce":2956018891},{"type":"v1","addr":"172.21.15.136:6813","nonce":2956018891}]},"public_addr":"172.21.15.136:6809/2956018891","cluster_addr":"172.21.15.136:6811/2956018891","heartbeat_back_addr":"172.21.15.136:6815/2956018891","heartbeat_front_addr":"172.21.15.136:6813/2956018891","state":["exists","up"]},{"osd":6,"uuid":"1e7c585f-b7eb-46a5-8109-2534928f990f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6816","nonce":1716373842},{"type":"v1","addr":"172.21.15.136:6817","nonce":1716373842}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6818","nonce":1716373842},{"type":"v1","addr":"172.21.15.136:6819","nonce":1716373842}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6822","nonce":1716373842},{"type":"v1","addr":"172.21.15.136:6823","nonce":1716373842}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6820","nonce":1716373842},{"type":"v1","addr":"172.21.15.136:6821","nonce":1716373842}]},"public_addr":"172.21.15.136:6817/1716373842","cluster_addr":"172.21.15.136:6819/1716373842","heartbeat_back_addr":"172.21.15.136:6823/1716373842","heartbeat_front_addr":"172.21.15.136:6821/1716373842","state":["exists","up"]},{"osd":7,"uuid":"504bffb3-f118-46e6-ab6c-bb6a807b29ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6824","nonce":3497474558},{"type":"v1","addr":"172.21.15.136:6825","nonce":3497474558}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6826","nonce":3497474558},{"type":"v1","addr":"172.21.15.136:6827","nonce":3497474558}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6830","nonce":3497474558},{"type":"v1","addr":"172.21.15.136:6831","nonce":3497474558}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6828","nonce":3497474558},{"type":"v1","addr":"172.21.15.136:6829","nonce":3497474558}]},"public_addr":"172.21.15.136:6825/3497474558","cluster_addr":"172.21.15.136:6827/3497474558","heartbeat_back_addr":"172.21.15.136:6831/3497474558","heartbeat_front_addr":"172.21.15.136:6829/3497474558","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T09:32:07.432854+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T09:32:39.270927+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T09:33:05.297405+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T09:33:31.722790+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T09:33:53.700594+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T09:34:18.634718+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T09:34:42.572076+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T09:35:07.388966+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.78:6801/3049272511":"2024-09-18T09:30:53.854987+0000","172.21.15.78:6800/3049272511":"2024-09-18T09:30:53.854987+0000","172.21.15.78:0/2217269551":"2024-09-18T09:30:53.854987+0000","172.21.15.78:0/3448812961":"2024-09-18T09:30:04.243833+0000","172.21.15.78:0/2050832447":"2024-09-18T09:30:04.243833+0000","172.21.15.78:0/2201892685":"2024-09-18T09:30:53.854987+0000","172.21.15.78:6801/3053679137":"2024-09-18T09:30:04.243833+0000","172.21.15.78:6800/3053679137":"2024-09-18T09:30:04.243833+0000","172.21.15.78:0/1896225568":"2024-09-18T09:30:53.854987+0000","172.21.15.78:0/1100384719":"2024-09-18T09:30:04.243833+0000","172.21.15.78:0/2571571226":"2024-09-18T09:29:33.964265+0000","172.21.15.78:6800/2661840943":"2024-09-18T09:29:33.964265+0000","172.21.15.78:6801/2661840943":"2024-09-18T09:29:33.964265+0000","172.21.15.78:0/2015147337":"2024-09-18T09:29:33.964265+0000","172.21.15.78:0/2030020465":"2024-09-18T09:29:33.964265+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-17T09:35:29.352 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-17T09:35:29.352 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-17T09:35:29.353 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-17T09:35:29.353 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-17T09:35:29.354 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-17T09:35:29.354 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-17T09:35:29.354 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-17T09:35:29.355 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-17T09:35:29.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:29 smithi078 ceph-mon[23689]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:29.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:29 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/1087458373' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T09:35:29.734 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-17T09:35:29.735 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-17T09:35:29.736 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-17T09:35:29.737 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-17T09:35:29.739 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-17T09:35:29.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:29 smithi136 ceph-mon[25953]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:29.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:29 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/1087458373' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T09:35:30.601 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-17T09:35:30.601 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-17T09:35:30.603 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-17T09:35:31.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:31 smithi078 ceph-mon[23689]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:31.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:31 smithi136 ceph-mon[25953]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:33.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:33 smithi078 ceph-mon[23689]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:33.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:33 smithi136 ceph-mon[25953]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:35.838 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:35 smithi078 ceph-mon[23689]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:35.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:35 smithi136 ceph-mon[25953]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:37.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:37 smithi078 ceph-mon[23689]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:37.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:37 smithi136 ceph-mon[25953]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:39.653 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:39 smithi078 ceph-mon[23689]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:39.766 INFO:teuthology.orchestra.run.smithi078.stdout:60129542183 2024-09-17T09:35:39.766 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-17T09:35:39.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:39 smithi136 ceph-mon[25953]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:40.167 INFO:teuthology.orchestra.run.smithi078.stdout:206158430222 2024-09-17T09:35:40.167 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-17T09:35:40.670 INFO:teuthology.orchestra.run.smithi078.stdout:81604378657 2024-09-17T09:35:40.671 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-17T09:35:41.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:41 smithi078 ceph-mon[23689]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:41.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:41 smithi136 ceph-mon[25953]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:43.836 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:43 smithi078 ceph-mon[23689]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:43.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:43 smithi136 ceph-mon[25953]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:44.306 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-17T09:35:44.722 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-17T09:35:44.722 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-17T09:35:45.173 INFO:teuthology.orchestra.run.smithi078.stdout:38654705710 2024-09-17T09:35:45.174 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-17T09:35:45.706 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:45 smithi078 ceph-mon[23689]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:45.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:45 smithi136 ceph-mon[25953]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:46.304 INFO:teuthology.orchestra.run.smithi078.stdout:137438953497 2024-09-17T09:35:46.304 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-17T09:35:47.369 INFO:teuthology.orchestra.run.smithi078.stdout:240518168586 2024-09-17T09:35:47.370 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-17T09:35:47.750 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:47 smithi078 ceph-mon[23689]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:47.772 INFO:teuthology.orchestra.run.smithi078.stdout:171798691859 2024-09-17T09:35:47.773 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-17T09:35:47.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:47 smithi136 ceph-mon[25953]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:48.316 INFO:teuthology.orchestra.run.smithi078.stdout:115964117020 2024-09-17T09:35:48.317 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-17T09:35:49.764 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:49 smithi078 ceph-mon[23689]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:49.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:49 smithi136 ceph-mon[25953]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:50.980 INFO:teuthology.orchestra.run.smithi078.stdout:206158430224 2024-09-17T09:35:51.200 INFO:teuthology.orchestra.run.smithi078.stdout:60129542185 2024-09-17T09:35:51.437 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-17T09:35:51.440 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-17T09:35:51.443 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-17T09:35:51.446 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-17T09:35:51.449 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-17T09:35:51.735 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:51 smithi078 ceph-mon[23689]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:51.736 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:51 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/2992842883' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-17T09:35:51.736 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:51 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/2952761403' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-17T09:35:51.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:51 smithi136 ceph-mon[25953]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:51.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:51 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/2992842883' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-17T09:35:51.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:51 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/2952761403' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-17T09:35:52.051 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542183 got 60129542185 for osd.1 2024-09-17T09:35:52.052 DEBUG:teuthology.parallel:result is None 2024-09-17T09:35:53.654 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430222 got 206158430224 for osd.6 2024-09-17T09:35:53.655 DEBUG:teuthology.parallel:result is None 2024-09-17T09:35:53.710 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:53 smithi078 ceph-mon[23689]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:53.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:53 smithi136 ceph-mon[25953]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:54.478 INFO:teuthology.orchestra.run.smithi078.stdout:81604378660 2024-09-17T09:35:54.686 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:54 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/2022626975' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-17T09:35:54.686 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:54 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/trash_purge_schedule"}]: dispatch 2024-09-17T09:35:54.686 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:54 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:35:54.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:54 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/2022626975' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-17T09:35:54.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:54 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/trash_purge_schedule"}]: dispatch 2024-09-17T09:35:54.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:54 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:35:55.221 INFO:teuthology.orchestra.run.smithi078.stdout:115964117022 2024-09-17T09:35:55.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:55 smithi078 ceph-mon[23689]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:55.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:55 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/1890977064' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-17T09:35:55.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:55 smithi136 ceph-mon[25953]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:56.011 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:55 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/1890977064' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-17T09:35:56.766 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378657 got 81604378660 for osd.2 2024-09-17T09:35:56.766 DEBUG:teuthology.parallel:result is None 2024-09-17T09:35:57.754 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:57 smithi078 ceph-mon[23689]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:57.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:57 smithi136 ceph-mon[25953]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:58.729 INFO:tasks.cephadm.ceph_manager.ceph:need seq 115964117020 got 115964117022 for osd.3 2024-09-17T09:35:58.729 DEBUG:teuthology.parallel:result is None 2024-09-17T09:35:59.776 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:35:59 smithi078 ceph-mon[23689]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:35:59.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:35:59 smithi136 ceph-mon[25953]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:00.662 INFO:teuthology.orchestra.run.smithi078.stdout:38654705714 2024-09-17T09:36:00.906 INFO:teuthology.orchestra.run.smithi078.stdout:240518168589 2024-09-17T09:36:01.748 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:01 smithi078 ceph-mon[23689]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:01.748 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:01 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/3895131656' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-17T09:36:01.749 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:01 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/1931279262' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-17T09:36:01.750 INFO:teuthology.orchestra.run.smithi078.stdout:171798691862 2024-09-17T09:36:01.842 INFO:teuthology.orchestra.run.smithi078.stdout:137438953500 2024-09-17T09:36:01.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:01 smithi136 ceph-mon[25953]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:01.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:01 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/3895131656' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-17T09:36:01.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:01 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/1931279262' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-17T09:36:02.460 INFO:tasks.cephadm.ceph_manager.ceph:need seq 137438953497 got 137438953500 for osd.4 2024-09-17T09:36:02.460 DEBUG:teuthology.parallel:result is None 2024-09-17T09:36:02.740 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:02 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/2023561385' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-17T09:36:02.740 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:02 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/4251970612' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-17T09:36:02.862 INFO:tasks.cephadm.ceph_manager.ceph:need seq 240518168586 got 240518168589 for osd.7 2024-09-17T09:36:02.862 DEBUG:teuthology.parallel:result is None 2024-09-17T09:36:02.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:02 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/2023561385' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-17T09:36:02.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:02 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/4251970612' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-17T09:36:03.312 INFO:tasks.cephadm.ceph_manager.ceph:need seq 171798691859 got 171798691862 for osd.5 2024-09-17T09:36:03.313 DEBUG:teuthology.parallel:result is None 2024-09-17T09:36:03.762 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705710 got 38654705714 for osd.0 2024-09-17T09:36:03.762 DEBUG:teuthology.parallel:result is None 2024-09-17T09:36:03.762 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-17T09:36:03.763 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-17T09:36:03.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:03 smithi078 ceph-mon[23689]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:03.970 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-17T09:36:03.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:03 smithi136 ceph-mon[25953]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:05.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:05 smithi078 ceph-mon[23689]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:05.874 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-17T09:36:05.876 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2024-09-17T09:36:05.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:05 smithi136 ceph-mon[25953]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:06.471 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":203,"stamp":"2024-09-17T09:36:04.513286+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":49016,"kb_used_data":6392,"kb_used_omap":0,"kb_used_meta":42560,"kb_avail":749912200,"statfs":{"total":767960285184,"available":767910092800,"internally_reserved":0,"allocated":6545408,"data_stored":3572920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43581440},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001928"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-17T09:35:12.562546+0000","last_change":"2024-09-17T09:35:12.562546+0000","last_active":"2024-09-17T09:35:12.562546+0000","last_peered":"2024-09-17T09:35:12.562546+0000","last_clean":"2024-09-17T09:35:12.562546+0000","last_became_active":"2024-09-17T09:35:12.562144+0000","last_became_peered":"2024-09-17T09:35:12.562144+0000","last_unstale":"2024-09-17T09:35:12.562546+0000","last_undegraded":"2024-09-17T09:35:12.562546+0000","last_fullsized":"2024-09-17T09:35:12.562546+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T09:33:09.483470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T09:33:09.483470+0000","last_clean_scrub_stamp":"2024-09-17T09:33:09.483470+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-18T14:23:12.767356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168590,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6060,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739092,"statfs":{"total":95995035648,"available":95988830208,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56499999999999995}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47899999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.442}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.502}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54400000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58899999999999997}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57899999999999996}]}]},{"osd":6,"up_from":48,"seq":206158430227,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6188,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738964,"statfs":{"total":95995035648,"available":95988699136,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 09:35:45 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.60099999999999998}]},{"osd":1,"last update":"Tue Sep 17 09:35:45 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.59099999999999997}]},{"osd":2,"last update":"Tue Sep 17 09:35:45 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.621},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.746}]},{"osd":3,"last update":"Tue Sep 17 09:35:45 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.67200000000000004}]},{"osd":4,"last update":"Tue Sep 17 09:35:45 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.49299999999999999}]},{"osd":5,"last update":"Tue Sep 17 09:35:45 2024","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.45500000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53600000000000003}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 09:35:45 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.33500000000000002,"15min":0.31},"min":{"1min":0.252,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"last":0.503},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.35799999999999998,"15min":0.33200000000000002},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.29799999999999999}]},{"osd":2,"last update":"Tue Sep 17 09:35:14 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.378,"15min":0.37},"min":{"1min":0.28599999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.31},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.39500000000000002,"15min":0.38400000000000001},"min":{"1min":0.35199999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.41399999999999998}]},{"osd":3,"last update":"Tue Sep 17 09:35:39 2024","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.45200000000000001,"15min":0.45800000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.53500000000000003,"5min":0.56599999999999995,"15min":0.56599999999999995},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.46800000000000003,"15min":0.47199999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.35599999999999998}]},{"osd":4,"last update":"Tue Sep 17 09:34:58 2024","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.998,"5min":0.998,"15min":0.998},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.58299999999999996}]},{"osd":5,"last update":"Tue Sep 17 09:35:24 2024","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.49199999999999999}]},{"osd":6,"last update":"Tue Sep 17 09:35:45 2024","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.96599999999999997,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.55200000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52600000000000002}]}]},{"osd":0,"up_from":9,"seq":38654705714,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6956,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738196,"statfs":{"total":95995035648,"available":95987912704,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Sep 17 09:35:47 2024","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.33200000000000002,"15min":0.30499999999999999},"min":{"1min":0.27800000000000002,"5min":0.215,"15min":0.215},"max":{"1min":0.41699999999999998,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.501},{"interface":"front","average":{"1min":0.36199999999999999,"5min":0.33600000000000002,"15min":0.30499999999999999},"min":{"1min":0.26000000000000001,"5min":0.221,"15min":0.221},"max":{"1min":0.52500000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.46700000000000003}]},{"osd":2,"last update":"Tue Sep 17 09:35:14 2024","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.36799999999999999,"15min":0.36399999999999999},"min":{"1min":0.27300000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.39200000000000002,"15min":0.38800000000000001},"min":{"1min":0.28499999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.59499999999999997}]},{"osd":3,"last update":"Tue Sep 17 09:35:35 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.44700000000000001,"15min":0.45000000000000001},"min":{"1min":0.34100000000000003,"5min":0.309,"15min":0.309},"max":{"1min":0.60399999999999998,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.44,"15min":0.44400000000000001},"min":{"1min":0.32200000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.60499999999999998,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.49199999999999999}]},{"osd":4,"last update":"Tue Sep 17 09:34:58 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.70499999999999996}]},{"osd":5,"last update":"Tue Sep 17 09:35:22 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.71399999999999997}]},{"osd":6,"last update":"Tue Sep 17 09:35:47 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.78000000000000003},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.67000000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73499999999999999}]}]},{"osd":2,"up_from":19,"seq":81604378663,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6308,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738844,"statfs":{"total":95995035648,"available":95988576256,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 09:35:08 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.36099999999999999,"15min":0.35799999999999998},"min":{"1min":0.26800000000000002,"5min":0.249,"15min":0.249},"max":{"1min":0.53900000000000003,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.371,"15min":0.36699999999999999},"min":{"1min":0.30099999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.55200000000000005}]},{"osd":1,"last update":"Tue Sep 17 09:35:08 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.36499999999999999,"15min":0.35999999999999999},"min":{"1min":0.28799999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.38200000000000001,"15min":0.376},"min":{"1min":0.32700000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.61399999999999999}]},{"osd":3,"last update":"Tue Sep 17 09:35:36 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.432,"15min":0.432},"min":{"1min":0.33300000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.435,"5min":0.441,"15min":0.442},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.58799999999999997}]},{"osd":4,"last update":"Tue Sep 17 09:35:58 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.60899999999999999,"15min":0.60499999999999998},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.80200000000000005,"5min":0.97099999999999997,"15min":0.97099999999999997},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.625,"5min":0.60099999999999998,"15min":0.59699999999999998},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.80700000000000005,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.68400000000000005}]},{"osd":5,"last update":"Tue Sep 17 09:35:21 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.67500000000000004}]},{"osd":6,"last update":"Tue Sep 17 09:35:45 2024","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.628}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73599999999999999}]}]},{"osd":3,"up_from":27,"seq":115964117024,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 09:35:39 2024","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.39500000000000002,"15min":0.39900000000000002},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.53900000000000003,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.317},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.432,"15min":0.434},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.41099999999999998}]},{"osd":1,"last update":"Tue Sep 17 09:35:39 2024","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.433,"15min":0.435},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41699999999999998,"15min":0.41799999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.51000000000000001}]},{"osd":2,"last update":"Tue Sep 17 09:35:39 2024","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.44600000000000001,"15min":0.44800000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.44500000000000001,"15min":0.44900000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.60699999999999998,"5min":0.623,"15min":0.623},"last":0.56499999999999995}]},{"osd":4,"last update":"Tue Sep 17 09:35:59 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":1.056,"5min":1.056,"15min":1.056},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.627,"15min":0.63100000000000001},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.78100000000000003,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.64600000000000002}]},{"osd":5,"last update":"Tue Sep 17 09:35:23 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.67500000000000004}]},{"osd":6,"last update":"Tue Sep 17 09:35:47 2024","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.47099999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.62}]}]},{"osd":4,"up_from":32,"seq":137438953501,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 09:35:55 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.498,"15min":0.48999999999999999},"min":{"1min":0.33700000000000002,"5min":0.318,"15min":0.318},"max":{"1min":0.73899999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.50600000000000001,"15min":0.5},"min":{"1min":0.36899999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.68500000000000005,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.38300000000000001}]},{"osd":1,"last update":"Tue Sep 17 09:35:55 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.50800000000000001,"15min":0.504},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.70999999999999996,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.51900000000000002,"15min":0.51200000000000001},"min":{"1min":0.39700000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.747,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.48699999999999999}]},{"osd":2,"last update":"Tue Sep 17 09:35:55 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.51800000000000002,"15min":0.51200000000000001},"min":{"1min":0.374,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.70299999999999996,"5min":0.877,"15min":0.877},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.504,"15min":0.496},"min":{"1min":0.42699999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.71499999999999997,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.42099999999999999}]},{"osd":3,"last update":"Tue Sep 17 09:35:55 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.51200000000000001,"15min":0.503},"min":{"1min":0.46700000000000003,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.68400000000000005,"5min":0.87,"15min":0.87},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.52400000000000002,"15min":0.51900000000000002},"min":{"1min":0.36099999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.73699999999999999,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.45100000000000001}]},{"osd":5,"last update":"Tue Sep 17 09:35:21 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.46700000000000003}]},{"osd":6,"last update":"Tue Sep 17 09:35:46 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.5},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.51700000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54000000000000004}]}]},{"osd":5,"up_from":40,"seq":171798691863,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5668,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739484,"statfs":{"total":95995035648,"available":95989231616,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 09:35:23 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.56100000000000005}]},{"osd":1,"last update":"Tue Sep 17 09:35:23 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.58499999999999996}]},{"osd":2,"last update":"Tue Sep 17 09:35:23 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.63},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.66300000000000003}]},{"osd":3,"last update":"Tue Sep 17 09:35:23 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.64600000000000002}]},{"osd":4,"last update":"Tue Sep 17 09:35:23 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.53500000000000003}]},{"osd":6,"last update":"Tue Sep 17 09:35:45 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.497}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57199999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-17T09:36:06.473 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-17T09:36:06.683 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-17T09:36:06.804 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:06 smithi078 ceph-mon[23689]: from='client.14446 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T09:36:06.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:06 smithi136 ceph-mon[25953]: from='client.14446 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T09:36:07.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:07 smithi078 ceph-mon[23689]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:07.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:07 smithi136 ceph-mon[25953]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:08.321 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-17T09:36:08.323 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2024-09-17T09:36:08.871 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":204,"stamp":"2024-09-17T09:36:06.513764+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":49016,"kb_used_data":6392,"kb_used_omap":0,"kb_used_meta":42560,"kb_avail":749912200,"statfs":{"total":767960285184,"available":767910092800,"internally_reserved":0,"allocated":6545408,"data_stored":3572920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43581440},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002108"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-17T09:35:12.562546+0000","last_change":"2024-09-17T09:35:12.562546+0000","last_active":"2024-09-17T09:35:12.562546+0000","last_peered":"2024-09-17T09:35:12.562546+0000","last_clean":"2024-09-17T09:35:12.562546+0000","last_became_active":"2024-09-17T09:35:12.562144+0000","last_became_peered":"2024-09-17T09:35:12.562144+0000","last_unstale":"2024-09-17T09:35:12.562546+0000","last_undegraded":"2024-09-17T09:35:12.562546+0000","last_fullsized":"2024-09-17T09:35:12.562546+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T09:33:09.483470+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T09:33:09.483470+0000","last_clean_scrub_stamp":"2024-09-17T09:33:09.483470+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-18T14:23:12.767356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168590,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6060,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739092,"statfs":{"total":95995035648,"available":95988830208,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56499999999999995}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47899999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.442}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.502}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54400000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58899999999999997}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57899999999999996}]}]},{"osd":6,"up_from":48,"seq":206158430228,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6188,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738964,"statfs":{"total":95995035648,"available":95988699136,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 09:35:45 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.5},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.46000000000000002}]},{"osd":1,"last update":"Tue Sep 17 09:35:45 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.54400000000000004}]},{"osd":2,"last update":"Tue Sep 17 09:35:45 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.55600000000000005}]},{"osd":3,"last update":"Tue Sep 17 09:35:45 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.59299999999999997}]},{"osd":4,"last update":"Tue Sep 17 09:35:45 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.61599999999999999}]},{"osd":5,"last update":"Tue Sep 17 09:35:45 2024","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.57599999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63900000000000001}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 09:35:45 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.33500000000000002,"15min":0.31},"min":{"1min":0.252,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"last":0.503},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.35799999999999998,"15min":0.33200000000000002},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.29799999999999999}]},{"osd":2,"last update":"Tue Sep 17 09:35:14 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.378,"15min":0.37},"min":{"1min":0.28599999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.31},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.39500000000000002,"15min":0.38400000000000001},"min":{"1min":0.35199999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.41399999999999998}]},{"osd":3,"last update":"Tue Sep 17 09:35:39 2024","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.45200000000000001,"15min":0.45800000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.53500000000000003,"5min":0.56599999999999995,"15min":0.56599999999999995},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.46800000000000003,"15min":0.47199999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.35599999999999998}]},{"osd":4,"last update":"Tue Sep 17 09:34:58 2024","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.998,"5min":0.998,"15min":0.998},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.58299999999999996}]},{"osd":5,"last update":"Tue Sep 17 09:35:24 2024","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.49199999999999999}]},{"osd":6,"last update":"Tue Sep 17 09:35:45 2024","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.96599999999999997,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.55200000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52600000000000002}]}]},{"osd":0,"up_from":9,"seq":38654705715,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6956,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738196,"statfs":{"total":95995035648,"available":95987912704,"internally_reserved":0,"allocated":1150976,"data_stored":777195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Sep 17 09:35:47 2024","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.33200000000000002,"15min":0.30499999999999999},"min":{"1min":0.27800000000000002,"5min":0.215,"15min":0.215},"max":{"1min":0.41699999999999998,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.36199999999999999,"5min":0.33600000000000002,"15min":0.30499999999999999},"min":{"1min":0.26000000000000001,"5min":0.221,"15min":0.221},"max":{"1min":0.52500000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.34699999999999998}]},{"osd":2,"last update":"Tue Sep 17 09:35:14 2024","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.36799999999999999,"15min":0.36399999999999999},"min":{"1min":0.27300000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.39200000000000002,"15min":0.38800000000000001},"min":{"1min":0.28499999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.317}]},{"osd":3,"last update":"Tue Sep 17 09:35:35 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.44700000000000001,"15min":0.45000000000000001},"min":{"1min":0.34100000000000003,"5min":0.309,"15min":0.309},"max":{"1min":0.60399999999999998,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.44,"15min":0.44400000000000001},"min":{"1min":0.32200000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.60499999999999998,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.375}]},{"osd":4,"last update":"Tue Sep 17 09:36:03 2024","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.60799999999999998,"15min":0.60199999999999998},"min":{"1min":0.45500000000000002,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.86499999999999999,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.60699999999999998,"15min":0.60199999999999998},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.871,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.69899999999999995}]},{"osd":5,"last update":"Tue Sep 17 09:35:22 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.54400000000000004}]},{"osd":6,"last update":"Tue Sep 17 09:35:47 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.753},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.55200000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60999999999999999}]}]},{"osd":2,"up_from":19,"seq":81604378663,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6308,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738844,"statfs":{"total":95995035648,"available":95988576256,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 09:35:08 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.36099999999999999,"15min":0.35799999999999998},"min":{"1min":0.26800000000000002,"5min":0.249,"15min":0.249},"max":{"1min":0.53900000000000003,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.371,"15min":0.36699999999999999},"min":{"1min":0.30099999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.55200000000000005}]},{"osd":1,"last update":"Tue Sep 17 09:35:08 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.36499999999999999,"15min":0.35999999999999999},"min":{"1min":0.28799999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.38200000000000001,"15min":0.376},"min":{"1min":0.32700000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.61399999999999999}]},{"osd":3,"last update":"Tue Sep 17 09:35:36 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.432,"15min":0.432},"min":{"1min":0.33300000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.435,"5min":0.441,"15min":0.442},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.58799999999999997}]},{"osd":4,"last update":"Tue Sep 17 09:35:58 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.60899999999999999,"15min":0.60499999999999998},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.80200000000000005,"5min":0.97099999999999997,"15min":0.97099999999999997},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.625,"5min":0.60099999999999998,"15min":0.59699999999999998},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.80700000000000005,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.68400000000000005}]},{"osd":5,"last update":"Tue Sep 17 09:35:21 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.67500000000000004}]},{"osd":6,"last update":"Tue Sep 17 09:35:45 2024","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.628}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73599999999999999}]}]},{"osd":3,"up_from":27,"seq":115964117025,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 09:35:39 2024","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.39500000000000002,"15min":0.39900000000000002},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.53900000000000003,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.317},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.432,"15min":0.434},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.41099999999999998}]},{"osd":1,"last update":"Tue Sep 17 09:35:39 2024","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.433,"15min":0.435},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41699999999999998,"15min":0.41799999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.51000000000000001}]},{"osd":2,"last update":"Tue Sep 17 09:35:39 2024","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.44600000000000001,"15min":0.44800000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.44500000000000001,"15min":0.44900000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.60699999999999998,"5min":0.623,"15min":0.623},"last":0.56499999999999995}]},{"osd":4,"last update":"Tue Sep 17 09:35:59 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":1.056,"5min":1.056,"15min":1.056},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.627,"15min":0.63100000000000001},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.78100000000000003,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.64600000000000002}]},{"osd":5,"last update":"Tue Sep 17 09:35:23 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.67500000000000004}]},{"osd":6,"last update":"Tue Sep 17 09:35:47 2024","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.47099999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.62}]}]},{"osd":4,"up_from":32,"seq":137438953502,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 09:35:55 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.498,"15min":0.48999999999999999},"min":{"1min":0.33700000000000002,"5min":0.318,"15min":0.318},"max":{"1min":0.73899999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.50600000000000001,"15min":0.5},"min":{"1min":0.36899999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.68500000000000005,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.63300000000000001}]},{"osd":1,"last update":"Tue Sep 17 09:35:55 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.50800000000000001,"15min":0.504},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.70999999999999996,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.51900000000000002,"15min":0.51200000000000001},"min":{"1min":0.39700000000000002,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.747,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.70199999999999996}]},{"osd":2,"last update":"Tue Sep 17 09:35:55 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.51800000000000002,"15min":0.51200000000000001},"min":{"1min":0.374,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.70299999999999996,"5min":0.877,"15min":0.877},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.504,"15min":0.496},"min":{"1min":0.42699999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.71499999999999997,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.57299999999999995}]},{"osd":3,"last update":"Tue Sep 17 09:35:55 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.51200000000000001,"15min":0.503},"min":{"1min":0.46700000000000003,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.68400000000000005,"5min":0.87,"15min":0.87},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.52400000000000002,"15min":0.51900000000000002},"min":{"1min":0.36099999999999999,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.73699999999999999,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.621}]},{"osd":5,"last update":"Tue Sep 17 09:35:21 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.436},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.51500000000000001}]},{"osd":6,"last update":"Tue Sep 17 09:35:46 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.64400000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67800000000000005}]}]},{"osd":5,"up_from":40,"seq":171798691863,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5668,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739484,"statfs":{"total":95995035648,"available":95989231616,"internally_reserved":0,"allocated":618496,"data_stored":248267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 09:35:23 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.56100000000000005}]},{"osd":1,"last update":"Tue Sep 17 09:35:23 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.58499999999999996}]},{"osd":2,"last update":"Tue Sep 17 09:35:23 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.63},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.66300000000000003}]},{"osd":3,"last update":"Tue Sep 17 09:35:23 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.64600000000000002}]},{"osd":4,"last update":"Tue Sep 17 09:35:23 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.53500000000000003}]},{"osd":6,"last update":"Tue Sep 17 09:35:45 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.497}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57199999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-17T09:36:08.873 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-17T09:36:08.873 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-17T09:36:08.873 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-17T09:36:08.873 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-17T09:36:09.079 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-17T09:36:09.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:09 smithi078 ceph-mon[23689]: from='client.14450 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T09:36:09.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:09 smithi078 ceph-mon[23689]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:09.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:09 smithi136 ceph-mon[25953]: from='client.14450 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T09:36:09.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:09 smithi136 ceph-mon[25953]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:10.959 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-17T09:36:10.959 INFO:teuthology.orchestra.run.smithi078.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-17T09:36:11.563 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-17T09:36:11.564 INFO:tasks.cephadm:Setup complete, yielding 2024-09-17T09:36:11.564 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-17T09:36:11.573 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-17T09:36:11.574 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- bash -c 'ceph orch status' 2024-09-17T09:36:11.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:11 smithi078 ceph-mon[23689]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:11.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:11 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/4078905945' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-17T09:36:11.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:11 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:36:11.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:11 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:36:11.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:11 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:36:11.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:11 smithi136 ceph-mon[25953]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:11.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:11 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/4078905945' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-17T09:36:11.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:11 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:36:11.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:11 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:36:11.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:11 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:36:12.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:12 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:13.005 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:12 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:13.503 INFO:teuthology.orchestra.run.smithi078.stdout:Backend: cephadm 2024-09-17T09:36:13.503 INFO:teuthology.orchestra.run.smithi078.stdout:Available: Yes 2024-09-17T09:36:13.504 INFO:teuthology.orchestra.run.smithi078.stdout:Paused: No 2024-09-17T09:36:13.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:13 smithi136 ceph-mon[25953]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:14.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:13 smithi078 ceph-mon[23689]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:14.210 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- bash -c 'ceph orch ps' 2024-09-17T09:36:14.797 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:14 smithi078 ceph-mon[23689]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:36:14.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:14 smithi136 ceph-mon[25953]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:36:15.755 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:15 smithi078 ceph-mon[23689]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:15.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:15 smithi136 ceph-mon[25953]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:16.177 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:36:16.177 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (4m) 2m ago 5m 15.5M - ba2b418f427c 8143c3c75418 2024-09-17T09:36:16.177 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (5m) 2m ago 5m 6400k - 17.2.0 e1d6a67b021e 46d44333624b 2024-09-17T09:36:16.177 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi136 smithi136 running (5m) 68s ago 5m 6395k - 17.2.0 e1d6a67b021e 56ce035cb482 2024-09-17T09:36:16.177 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (4m) 2m ago 5m 45.5M - 8.3.5 dad864ee21e9 27395d6baf92 2024-09-17T09:36:16.177 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rsqofi smithi078 *:9283 running (7m) 2m ago 7m 447M - 17.2.0 e1d6a67b021e f2012c57d1ac 2024-09-17T09:36:16.177 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi136.hicmlm smithi136 *:8443,9283 running (5m) 68s ago 5m 412M - 17.2.0 e1d6a67b021e b12edb7834e5 2024-09-17T09:36:16.177 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (7m) 2m ago 7m 44.7M 2048M 17.2.0 e1d6a67b021e 09299430e248 2024-09-17T09:36:16.177 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi136 smithi136 running (5m) 68s ago 5m 39.5M 2048M 17.2.0 e1d6a67b021e 1bfdfb0617bb 2024-09-17T09:36:16.178 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (5m) 2m ago 5m 18.8M - 1dbe0e931976 ca57ddb194b4 2024-09-17T09:36:16.178 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi136 smithi136 *:9100 running (4m) 68s ago 4m 17.6M - 1dbe0e931976 7205c98b1bae 2024-09-17T09:36:16.178 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (4m) 2m ago 4m 32.7M 3216M 17.2.0 e1d6a67b021e b4ca42f8477d 2024-09-17T09:36:16.178 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (3m) 2m ago 3m 34.7M 3216M 17.2.0 e1d6a67b021e 351873014de4 2024-09-17T09:36:16.178 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (3m) 2m ago 3m 31.2M 3216M 17.2.0 e1d6a67b021e 07eb71e4fa57 2024-09-17T09:36:16.178 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (2m) 2m ago 2m 10.6M 3216M 17.2.0 e1d6a67b021e 1b3a8be89881 2024-09-17T09:36:16.178 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi136 running (2m) 68s ago 2m 33.1M 3984M 17.2.0 e1d6a67b021e a0c5b4de2414 2024-09-17T09:36:16.178 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi136 running (2m) 68s ago 2m 32.6M 3984M 17.2.0 e1d6a67b021e 30a3049955f9 2024-09-17T09:36:16.178 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi136 running (98s) 68s ago 97s 31.2M 3984M 17.2.0 e1d6a67b021e 1698e1567c33 2024-09-17T09:36:16.178 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi136 running (73s) 68s ago 72s 10.9M 3984M 17.2.0 e1d6a67b021e 5d3b08cecd78 2024-09-17T09:36:16.178 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (4m) 2m ago 4m 51.9M - 514e6a882f6e 8405bb1fd6e6 2024-09-17T09:36:16.768 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- bash -c 'ceph orch ls' 2024-09-17T09:36:17.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:17 smithi136 ceph-mon[25953]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:36:17.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:17 smithi136 ceph-mon[25953]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:18.005 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:17 smithi078 ceph-mon[23689]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:36:18.005 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:17 smithi078 ceph-mon[23689]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:18.697 INFO:teuthology.orchestra.run.smithi078.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-09-17T09:36:18.697 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-09-17T09:36:18.697 INFO:teuthology.orchestra.run.smithi078.stdout:crash 2/2 2m ago 6m * 2024-09-17T09:36:18.697 INFO:teuthology.orchestra.run.smithi078.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-09-17T09:36:18.698 INFO:teuthology.orchestra.run.smithi078.stdout:mgr 2/2 2m ago 6m count:2 2024-09-17T09:36:18.698 INFO:teuthology.orchestra.run.smithi078.stdout:mon 2/2 2m ago 5m smithi078:172.21.15.78=smithi078;smithi136:172.21.15.136=smithi136;count:2 2024-09-17T09:36:18.698 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-09-17T09:36:18.698 INFO:teuthology.orchestra.run.smithi078.stdout:osd 8 2m ago - 2024-09-17T09:36:18.698 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-09-17T09:36:19.386 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- bash -c 'ceph orch host ls' 2024-09-17T09:36:19.975 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:19 smithi078 ceph-mon[23689]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:19.976 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:19 smithi078 ceph-mon[23689]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:36:19.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:19 smithi136 ceph-mon[25953]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:19.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:19 smithi136 ceph-mon[25953]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:36:21.258 INFO:teuthology.orchestra.run.smithi078.stdout:HOST ADDR LABELS STATUS 2024-09-17T09:36:21.259 INFO:teuthology.orchestra.run.smithi078.stdout:smithi078 172.21.15.78 2024-09-17T09:36:21.259 INFO:teuthology.orchestra.run.smithi078.stdout:smithi136 172.21.15.136 2024-09-17T09:36:21.259 INFO:teuthology.orchestra.run.smithi078.stdout:2 hosts in cluster 2024-09-17T09:36:21.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:21 smithi078 ceph-mon[23689]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:21.893 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- bash -c 'ceph orch device ls' 2024-09-17T09:36:21.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:21 smithi136 ceph-mon[25953]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:22.841 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:22 smithi078 ceph-mon[23689]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:36:22.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:22 smithi136 ceph-mon[25953]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:36:23.832 INFO:teuthology.orchestra.run.smithi078.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-09-17T09:36:23.832 INFO:teuthology.orchestra.run.smithi078.stdout:smithi078 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620400QR400BGN 400G LVM detected, locked 2024-09-17T09:36:23.832 INFO:teuthology.orchestra.run.smithi078.stdout:smithi136 /dev/nvme0n1 ssd INTEL_SSDPEDMX020T4_CVPD514100122P0SGN 2000G LVM detected, locked 2024-09-17T09:36:23.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:23 smithi136 ceph-mon[25953]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:24.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:23 smithi078 ceph-mon[23689]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:24.581 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-17T09:36:24.593 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-17T09:36:24.594 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-17T09:36:24.620 INFO:teuthology.orchestra.run.smithi078.stderr:+ systemctl stop nfs-server 2024-09-17T09:36:24.628 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi136.front.sepia.ceph.com 2024-09-17T09:36:24.628 DEBUG:teuthology.orchestra.run.smithi136:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-17T09:36:24.659 INFO:teuthology.orchestra.run.smithi136.stderr:+ systemctl stop nfs-server 2024-09-17T09:36:24.668 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-17T09:36:24.676 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-17T09:36:24.677 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- bash -c 'ceph fs volume create foofs' 2024-09-17T09:36:24.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:24 smithi078 ceph-mon[23689]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:36:24.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:24 smithi136 ceph-mon[25953]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:36:25.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:25 smithi136 ceph-mon[25953]: pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:25.996 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:25 smithi078 ceph-mon[23689]: pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:27.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:27 smithi136 ceph-mon[25953]: pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:27.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:27 smithi136 ceph-mon[25953]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:36:27.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:27 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-17T09:36:28.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:27 smithi078 ceph-mon[23689]: pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:28.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:27 smithi078 ceph-mon[23689]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:36:28.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:27 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-17T09:36:29.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:28 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-17T09:36:29.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:28 smithi078 ceph-mon[23689]: osdmap e61: 8 total, 8 up, 8 in 2024-09-17T09:36:29.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:28 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-17T09:36:29.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:28 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi078[23685]: 2024-09-17T09:36:28.737+0000 7ff8df61f700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T09:36:29.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:28 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-17T09:36:29.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:28 smithi136 ceph-mon[25953]: osdmap e61: 8 total, 8 up, 8 in 2024-09-17T09:36:29.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:28 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-17T09:36:29.502 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-09-17T09:36:29.510 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-09-17T09:36:29.510 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-17T09:36:29.825 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:29 smithi078 ceph-mon[23689]: pgmap v216: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:29.825 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:29 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-17T09:36:29.825 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:29 smithi078 ceph-mon[23689]: osdmap e62: 8 total, 8 up, 8 in 2024-09-17T09:36:29.825 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:29 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-17T09:36:29.825 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:29 smithi078 ceph-mon[23689]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T09:36:29.825 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:29 smithi078 ceph-mon[23689]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-17T09:36:29.826 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:29 smithi078 ceph-mon[23689]: osdmap e63: 8 total, 8 up, 8 in 2024-09-17T09:36:29.826 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:29 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-17T09:36:29.826 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:29 smithi078 ceph-mon[23689]: fsmap foofs:0 2024-09-17T09:36:29.826 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:29 smithi078 ceph-mon[23689]: Saving service mds.foofs spec with placement count:2 2024-09-17T09:36:29.826 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:29 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:29.826 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:29 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:36:29.826 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:29 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:36:29.826 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:29 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:36:29.826 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:29 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:29.826 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:29 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:29.826 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:29 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.jdrsup", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T09:36:29.826 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:29 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.jdrsup", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-17T09:36:29.826 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:29 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:36:29.827 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:29 smithi078 ceph-mon[23689]: Deploying daemon mds.foofs.smithi078.jdrsup on smithi078 2024-09-17T09:36:30.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:29 smithi136 ceph-mon[25953]: pgmap v216: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:30.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:29 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-17T09:36:30.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:29 smithi136 ceph-mon[25953]: osdmap e62: 8 total, 8 up, 8 in 2024-09-17T09:36:30.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:29 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-17T09:36:30.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:29 smithi136 ceph-mon[25953]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T09:36:30.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:29 smithi136 ceph-mon[25953]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-17T09:36:30.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:29 smithi136 ceph-mon[25953]: osdmap e63: 8 total, 8 up, 8 in 2024-09-17T09:36:30.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:29 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-17T09:36:30.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:29 smithi136 ceph-mon[25953]: fsmap foofs:0 2024-09-17T09:36:30.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:29 smithi136 ceph-mon[25953]: Saving service mds.foofs spec with placement count:2 2024-09-17T09:36:30.228 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:29 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:30.228 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:29 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:36:30.228 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:29 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:36:30.228 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:29 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:36:30.228 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:29 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:30.228 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:29 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:30.229 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:29 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.jdrsup", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T09:36:30.229 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:29 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.jdrsup", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-17T09:36:30.229 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:29 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:36:30.229 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:29 smithi136 ceph-mon[25953]: Deploying daemon mds.foofs.smithi078.jdrsup on smithi078 2024-09-17T09:36:31.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:30 smithi078 ceph-mon[23689]: osdmap e64: 8 total, 8 up, 8 in 2024-09-17T09:36:31.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:30 smithi136 ceph-mon[25953]: osdmap e64: 8 total, 8 up, 8 in 2024-09-17T09:36:32.024 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:31 smithi078 ceph-mon[23689]: pgmap v220: 65 pgs: 6 creating+peering, 41 unknown, 18 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:32.024 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:31 smithi078 ceph-mon[23689]: osdmap e65: 8 total, 8 up, 8 in 2024-09-17T09:36:32.102 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-17T09:36:32.102 INFO:teuthology.orchestra.run.smithi078.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-17T09:29:53.798273Z", "last_refresh": "2024-09-17T09:33:34.200275Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-17T09:29:47.463665Z", "last_refresh": "2024-09-17T09:33:34.200338Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-17T09:29:51.372259Z", "last_refresh": "2024-09-17T09:33:34.200398Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-17T09:36:28.758783Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-17T09:36:28.753750Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-17T09:29:45.312225Z", "last_refresh": "2024-09-17T09:33:34.200205Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi078:172.21.15.78=smithi078", "smithi136:172.21.15.136=smithi136"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-17T09:30:35.435397Z", "last_refresh": "2024-09-17T09:33:34.200050Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-17T09:29:52.565464Z", "last_refresh": "2024-09-17T09:33:34.200457Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-17T09:33:34.200576Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-17T09:29:49.143959Z", "last_refresh": "2024-09-17T09:33:34.200516Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-17T09:36:32.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:31 smithi136 ceph-mon[25953]: pgmap v220: 65 pgs: 6 creating+peering, 41 unknown, 18 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:32.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:31 smithi136 ceph-mon[25953]: osdmap e65: 8 total, 8 up, 8 in 2024-09-17T09:36:33.233 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-17T09:36:34.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:33 smithi078 ceph-mon[23689]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T09:36:34.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:33 smithi078 ceph-mon[23689]: pgmap v222: 65 pgs: 6 creating+peering, 10 unknown, 49 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:34.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:33 smithi136 ceph-mon[25953]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T09:36:34.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:33 smithi136 ceph-mon[25953]: pgmap v222: 65 pgs: 6 creating+peering, 10 unknown, 49 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:34.234 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-17T09:36:35.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:34 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:35.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:34 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi136.smslff", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T09:36:35.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:34 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi136.smslff", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-17T09:36:35.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:34 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:36:35.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:34 smithi136 ceph-mon[25953]: Deploying daemon mds.foofs.smithi136.smslff on smithi136 2024-09-17T09:36:35.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:34 smithi136 ceph-mon[25953]: mds.? [v2:172.21.15.78:6834/2954767313,v1:172.21.15.78:6835/2954767313] up:boot 2024-09-17T09:36:35.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:34 smithi136 ceph-mon[25953]: daemon mds.foofs.smithi078.jdrsup assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-17T09:36:35.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:34 smithi136 ceph-mon[25953]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-17T09:36:35.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:34 smithi136 ceph-mon[25953]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-17T09:36:35.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:34 smithi136 ceph-mon[25953]: Cluster is now healthy 2024-09-17T09:36:35.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:34 smithi136 ceph-mon[25953]: fsmap foofs:0 1 up:standby 2024-09-17T09:36:35.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:34 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.jdrsup"}]: dispatch 2024-09-17T09:36:35.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:34 smithi136 ceph-mon[25953]: fsmap foofs:1 {0=foofs.smithi078.jdrsup=up:creating} 2024-09-17T09:36:35.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:34 smithi136 ceph-mon[25953]: daemon mds.foofs.smithi078.jdrsup is now active in filesystem foofs as rank 0 2024-09-17T09:36:35.242 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:34 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:35.242 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:34 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi136.smslff", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T09:36:35.242 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:34 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi136.smslff", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-17T09:36:35.242 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:34 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:36:35.242 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:34 smithi078 ceph-mon[23689]: Deploying daemon mds.foofs.smithi136.smslff on smithi136 2024-09-17T09:36:35.242 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:34 smithi078 ceph-mon[23689]: mds.? [v2:172.21.15.78:6834/2954767313,v1:172.21.15.78:6835/2954767313] up:boot 2024-09-17T09:36:35.242 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:34 smithi078 ceph-mon[23689]: daemon mds.foofs.smithi078.jdrsup assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-17T09:36:35.242 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:34 smithi078 ceph-mon[23689]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-17T09:36:35.242 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:34 smithi078 ceph-mon[23689]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-17T09:36:35.242 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:34 smithi078 ceph-mon[23689]: Cluster is now healthy 2024-09-17T09:36:35.242 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:34 smithi078 ceph-mon[23689]: fsmap foofs:0 1 up:standby 2024-09-17T09:36:35.242 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:34 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.jdrsup"}]: dispatch 2024-09-17T09:36:35.242 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:34 smithi078 ceph-mon[23689]: fsmap foofs:1 {0=foofs.smithi078.jdrsup=up:creating} 2024-09-17T09:36:35.243 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:34 smithi078 ceph-mon[23689]: daemon mds.foofs.smithi078.jdrsup is now active in filesystem foofs as rank 0 2024-09-17T09:36:35.953 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:35 smithi078 ceph-mon[23689]: pgmap v223: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:35.953 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:35 smithi078 ceph-mon[23689]: mds.? [v2:172.21.15.78:6834/2954767313,v1:172.21.15.78:6835/2954767313] up:active 2024-09-17T09:36:35.953 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:35 smithi078 ceph-mon[23689]: fsmap foofs:1 {0=foofs.smithi078.jdrsup=up:active} 2024-09-17T09:36:35.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:35 smithi136 ceph-mon[25953]: pgmap v223: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-17T09:36:35.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:35 smithi136 ceph-mon[25953]: mds.? [v2:172.21.15.78:6834/2954767313,v1:172.21.15.78:6835/2954767313] up:active 2024-09-17T09:36:35.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:35 smithi136 ceph-mon[25953]: fsmap foofs:1 {0=foofs.smithi078.jdrsup=up:active} 2024-09-17T09:36:36.362 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-17T09:36:36.362 INFO:teuthology.orchestra.run.smithi078.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-17T09:29:53.798273Z", "last_refresh": "2024-09-17T09:33:34.200275Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-17T09:29:47.463665Z", "last_refresh": "2024-09-17T09:33:34.200338Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-17T09:29:51.372259Z", "last_refresh": "2024-09-17T09:33:34.200398Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-17T09:36:28.758783Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-17T09:36:28.753750Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-17T09:29:45.312225Z", "last_refresh": "2024-09-17T09:33:34.200205Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi078:172.21.15.78=smithi078", "smithi136:172.21.15.136=smithi136"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-17T09:30:35.435397Z", "last_refresh": "2024-09-17T09:33:34.200050Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-17T09:29:52.565464Z", "last_refresh": "2024-09-17T09:33:34.200457Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-17T09:33:34.200576Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-17T09:29:49.143959Z", "last_refresh": "2024-09-17T09:33:34.200516Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-17T09:36:37.123 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-17T09:36:37.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:37 smithi136 ceph-mon[25953]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T09:36:37.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:37 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:37.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:37 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:36:37.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:37 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:36:37.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:37 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:36:37.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:37 smithi136 ceph-mon[25953]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 2 op/s 2024-09-17T09:36:37.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:37 smithi136 ceph-mon[25953]: mds.? [v2:172.21.15.136:6832/1260769482,v1:172.21.15.136:6833/1260769482] up:boot 2024-09-17T09:36:37.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:37 smithi136 ceph-mon[25953]: fsmap foofs:1 {0=foofs.smithi078.jdrsup=up:active} 1 up:standby 2024-09-17T09:36:37.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:37 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi136.smslff"}]: dispatch 2024-09-17T09:36:37.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:37 smithi136 ceph-mon[25953]: fsmap foofs:1 {0=foofs.smithi078.jdrsup=up:active} 1 up:standby 2024-09-17T09:36:37.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:37 smithi078 ceph-mon[23689]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T09:36:37.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:37 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:37.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:37 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:36:37.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:37 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:36:37.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:37 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:36:37.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:37 smithi078 ceph-mon[23689]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 2 op/s 2024-09-17T09:36:37.787 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:37 smithi078 ceph-mon[23689]: mds.? [v2:172.21.15.136:6832/1260769482,v1:172.21.15.136:6833/1260769482] up:boot 2024-09-17T09:36:37.787 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:37 smithi078 ceph-mon[23689]: fsmap foofs:1 {0=foofs.smithi078.jdrsup=up:active} 1 up:standby 2024-09-17T09:36:37.787 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:37 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi136.smslff"}]: dispatch 2024-09-17T09:36:37.787 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:37 smithi078 ceph-mon[23689]: fsmap foofs:1 {0=foofs.smithi078.jdrsup=up:active} 1 up:standby 2024-09-17T09:36:38.124 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-17T09:36:39.538 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:39 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:39.539 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:39 smithi078 ceph-mon[23689]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2024-09-17T09:36:39.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:39 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:39.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:39 smithi136 ceph-mon[25953]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2024-09-17T09:36:40.618 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-17T09:36:40.618 INFO:teuthology.orchestra.run.smithi078.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-17T09:29:53.798273Z", "last_refresh": "2024-09-17T09:33:34.200275Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-17T09:29:47.463665Z", "last_refresh": "2024-09-17T09:33:34.200338Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-17T09:29:51.372259Z", "last_refresh": "2024-09-17T09:33:34.200398Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-17T09:36:28.758783Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-17T09:36:28.753750Z", "last_refresh": "2024-09-17T09:36:38.294796Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-17T09:29:45.312225Z", "last_refresh": "2024-09-17T09:33:34.200205Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi078:172.21.15.78=smithi078", "smithi136:172.21.15.136=smithi136"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-17T09:30:35.435397Z", "last_refresh": "2024-09-17T09:33:34.200050Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-17T09:29:52.565464Z", "last_refresh": "2024-09-17T09:33:34.200457Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-17T09:33:34.200576Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-17T09:29:49.143959Z", "last_refresh": "2024-09-17T09:33:34.200516Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-17T09:36:40.810 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:40 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:40.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:40 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:41.656 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:41 smithi078 ceph-mon[23689]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-17T09:36:41.657 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:41 smithi078 ceph-mon[23689]: from='client.14498 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T09:36:41.708 INFO:tasks.cephadm:mds.foofs has 1/2 2024-09-17T09:36:41.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:41 smithi136 ceph-mon[25953]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-17T09:36:41.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:41 smithi136 ceph-mon[25953]: from='client.14498 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T09:36:42.709 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-17T09:36:43.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:43 smithi078 ceph-mon[23689]: pgmap v227: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-17T09:36:43.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:43 smithi136 ceph-mon[25953]: pgmap v227: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-17T09:36:44.838 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-17T09:36:44.838 INFO:teuthology.orchestra.run.smithi078.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-17T09:29:53.798273Z", "last_refresh": "2024-09-17T09:36:44.723047Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-17T09:29:47.463665Z", "last_refresh": "2024-09-17T09:36:38.294187Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-17T09:29:51.372259Z", "last_refresh": "2024-09-17T09:36:44.723177Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-17T09:36:28.758783Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-17T09:36:28.753750Z", "last_refresh": "2024-09-17T09:36:38.294796Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-17T09:29:45.312225Z", "last_refresh": "2024-09-17T09:36:38.294332Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi078:172.21.15.78=smithi078", "smithi136:172.21.15.136=smithi136"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-17T09:30:35.435397Z", "last_refresh": "2024-09-17T09:36:38.294406Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-17T09:29:52.565464Z", "last_refresh": "2024-09-17T09:36:38.294478Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-17T09:36:38.294546Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-17T09:29:49.143959Z", "last_refresh": "2024-09-17T09:36:44.723303Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-17T09:36:45.554 INFO:tasks.cephadm:mds.foofs has 2/2 2024-09-17T09:36:45.554 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-17T09:36:45.563 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-17T09:36:45.564 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-09-17T09:36:45.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:45 smithi078 ceph-mon[23689]: pgmap v228: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-17T09:36:45.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:45 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:45.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:45 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:45.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:45 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:36:45.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:45 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:36:45.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:45 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:36:45.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:45 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:45.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:45 smithi078 ceph-mon[23689]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T09:36:46.022 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:45 smithi136 ceph-mon[25953]: pgmap v228: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-17T09:36:46.022 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:45 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:46.022 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:45 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:46.023 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:45 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:36:46.023 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:45 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:36:46.023 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:45 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:36:46.023 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:45 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:46.023 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:45 smithi136 ceph-mon[25953]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T09:36:48.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:47 smithi078 ceph-mon[23689]: pgmap v229: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-17T09:36:48.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:47 smithi136 ceph-mon[25953]: pgmap v229: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-17T09:36:48.933 INFO:teuthology.orchestra.run.smithi078.stdout:NFS Cluster Created Successfully 2024-09-17T09:36:49.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:48 smithi078 ceph-mon[23689]: from='client.14506 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:36:49.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:48 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-17T09:36:49.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:48 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-17T09:36:49.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:48 smithi078 ceph-mon[23689]: osdmap e66: 8 total, 8 up, 8 in 2024-09-17T09:36:49.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:48 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-17T09:36:49.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:48 smithi136 ceph-mon[25953]: from='client.14506 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:36:49.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:48 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-17T09:36:49.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:48 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-17T09:36:49.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:48 smithi136 ceph-mon[25953]: osdmap e66: 8 total, 8 up, 8 in 2024-09-17T09:36:49.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:48 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-17T09:36:49.747 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-09-17T09:36:49.975 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:49 smithi078 ceph-mon[23689]: pgmap v231: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-17T09:36:49.976 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-17T09:36:49.976 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:49 smithi078 ceph-mon[23689]: osdmap e67: 8 total, 8 up, 8 in 2024-09-17T09:36:49.976 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:49.976 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:36:49.976 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:36:49.976 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:36:49.976 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:49.976 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:49.976 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.tcwbvi", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-17T09:36:49.976 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.tcwbvi", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-17T09:36:49.977 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-17T09:36:49.977 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T09:36:49.977 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:36:49.977 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-17T09:36:49.977 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T09:36:49.977 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.tcwbvi-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T09:36:49.977 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.tcwbvi-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-17T09:36:49.977 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:49 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:36:50.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:49 smithi136 ceph-mon[25953]: pgmap v231: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-17T09:36:50.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-17T09:36:50.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:49 smithi136 ceph-mon[25953]: osdmap e67: 8 total, 8 up, 8 in 2024-09-17T09:36:50.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:50.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:36:50.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:36:50.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:36:50.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:50.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:50.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.tcwbvi", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-17T09:36:50.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.tcwbvi", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-17T09:36:50.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-17T09:36:50.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T09:36:50.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:36:50.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-17T09:36:50.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T09:36:50.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.tcwbvi-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T09:36:50.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.tcwbvi-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-17T09:36:50.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:49 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:36:51.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:50 smithi136 ceph-mon[25953]: Saving service nfs.foo spec with placement count:2 2024-09-17T09:36:51.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:50 smithi136 ceph-mon[25953]: Creating key for client.nfs.foo.0.0.smithi078.tcwbvi 2024-09-17T09:36:51.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:50 smithi136 ceph-mon[25953]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-17T09:36:51.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:50 smithi136 ceph-mon[25953]: Rados config object exists: conf-nfs.foo 2024-09-17T09:36:51.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:50 smithi136 ceph-mon[25953]: Creating key for client.nfs.foo.0.0.smithi078.tcwbvi-rgw 2024-09-17T09:36:51.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:50 smithi136 ceph-mon[25953]: Deploying daemon nfs.foo.0.0.smithi078.tcwbvi on smithi078 2024-09-17T09:36:51.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:50 smithi136 ceph-mon[25953]: osdmap e68: 8 total, 8 up, 8 in 2024-09-17T09:36:51.231 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:50 smithi078 ceph-mon[23689]: Saving service nfs.foo spec with placement count:2 2024-09-17T09:36:51.231 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:50 smithi078 ceph-mon[23689]: Creating key for client.nfs.foo.0.0.smithi078.tcwbvi 2024-09-17T09:36:51.231 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:50 smithi078 ceph-mon[23689]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-17T09:36:51.231 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:50 smithi078 ceph-mon[23689]: Rados config object exists: conf-nfs.foo 2024-09-17T09:36:51.231 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:50 smithi078 ceph-mon[23689]: Creating key for client.nfs.foo.0.0.smithi078.tcwbvi-rgw 2024-09-17T09:36:51.231 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:50 smithi078 ceph-mon[23689]: Deploying daemon nfs.foo.0.0.smithi078.tcwbvi on smithi078 2024-09-17T09:36:51.232 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:50 smithi078 ceph-mon[23689]: osdmap e68: 8 total, 8 up, 8 in 2024-09-17T09:36:52.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:51 smithi136 ceph-mon[25953]: pgmap v234: 97 pgs: 20 unknown, 77 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-17T09:36:52.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:51 smithi136 ceph-mon[25953]: mgrmap e20: smithi078.rsqofi(active, since 5m), standbys: smithi136.hicmlm 2024-09-17T09:36:52.264 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:51 smithi078 ceph-mon[23689]: pgmap v234: 97 pgs: 20 unknown, 77 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-17T09:36:52.265 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:51 smithi078 ceph-mon[23689]: mgrmap e20: smithi078.rsqofi(active, since 5m), standbys: smithi136.hicmlm 2024-09-17T09:36:53.022 INFO:teuthology.orchestra.run.smithi078.stderr:Invalid command: Unexpected argument '--clusterid' 2024-09-17T09:36:53.023 INFO:teuthology.orchestra.run.smithi078.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-09-17T09:36:53.023 INFO:teuthology.orchestra.run.smithi078.stderr:Error EINVAL: invalid command 2024-09-17T09:36:53.380 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:36:53.380 INFO:teuthology.orchestra.run.smithi078.stdout: "bind": "/fake", 2024-09-17T09:36:53.380 INFO:teuthology.orchestra.run.smithi078.stdout: "fs": "foofs", 2024-09-17T09:36:53.380 INFO:teuthology.orchestra.run.smithi078.stdout: "path": "/", 2024-09-17T09:36:53.380 INFO:teuthology.orchestra.run.smithi078.stdout: "cluster": "foo", 2024-09-17T09:36:53.380 INFO:teuthology.orchestra.run.smithi078.stdout: "mode": "RW" 2024-09-17T09:36:53.381 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:36:54.043 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:53 smithi078 ceph-mon[23689]: pgmap v235: 97 pgs: 1 unknown, 96 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-17T09:36:54.044 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:53 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:54.044 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:53 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi136.regjvc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-17T09:36:54.044 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:53 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi136.regjvc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-17T09:36:54.044 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:53 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-17T09:36:54.044 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:53 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T09:36:54.044 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:53 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:36:54.044 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:53 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-17T09:36:54.044 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:53 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-17T09:36:54.066 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-09-17T09:36:54.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:53 smithi136 ceph-mon[25953]: pgmap v235: 97 pgs: 1 unknown, 96 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-17T09:36:54.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:53 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:54.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:53 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi136.regjvc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-17T09:36:54.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:53 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi136.regjvc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-17T09:36:54.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:53 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-17T09:36:54.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:53 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T09:36:54.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:53 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:36:54.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:53 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-17T09:36:54.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:53 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-17T09:36:55.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:54 smithi136 ceph-mon[25953]: Creating key for client.nfs.foo.1.0.smithi136.regjvc 2024-09-17T09:36:55.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:54 smithi136 ceph-mon[25953]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-17T09:36:55.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:54 smithi136 ceph-mon[25953]: from='client.14528 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:36:55.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:54 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:36:55.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:54 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/trash_purge_schedule"}]: dispatch 2024-09-17T09:36:55.337 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:54 smithi078 ceph-mon[23689]: Creating key for client.nfs.foo.1.0.smithi136.regjvc 2024-09-17T09:36:55.337 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:54 smithi078 ceph-mon[23689]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-17T09:36:55.337 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:54 smithi078 ceph-mon[23689]: from='client.14528 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:36:55.337 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:54 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:36:55.337 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:54 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/trash_purge_schedule"}]: dispatch 2024-09-17T09:36:56.219 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:55 smithi078 ceph-mon[23689]: pgmap v236: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 154 B/s rd, 308 B/s wr, 0 op/s 2024-09-17T09:36:56.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:55 smithi136 ceph-mon[25953]: pgmap v236: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 154 B/s rd, 308 B/s wr, 0 op/s 2024-09-17T09:36:56.944 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:56 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-17T09:36:56.945 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:56 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T09:36:56.945 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:56 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi136.regjvc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T09:36:56.945 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:56 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi136.regjvc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-17T09:36:56.945 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:56 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:36:57.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:56 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-17T09:36:57.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:56 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T09:36:57.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:56 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi136.regjvc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T09:36:57.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:56 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi136.regjvc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-17T09:36:57.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:56 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:36:57.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:57 smithi136 ceph-mon[25953]: from='client.14544 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:36:57.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:57 smithi136 ceph-mon[25953]: Rados config object exists: conf-nfs.foo 2024-09-17T09:36:57.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:57 smithi136 ceph-mon[25953]: Creating key for client.nfs.foo.1.0.smithi136.regjvc-rgw 2024-09-17T09:36:57.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:57 smithi136 ceph-mon[25953]: Deploying daemon nfs.foo.1.0.smithi136.regjvc on smithi136 2024-09-17T09:36:57.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:57 smithi136 ceph-mon[25953]: pgmap v237: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-09-17T09:36:58.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:57 smithi078 ceph-mon[23689]: from='client.14544 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:36:58.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:57 smithi078 ceph-mon[23689]: Rados config object exists: conf-nfs.foo 2024-09-17T09:36:58.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:57 smithi078 ceph-mon[23689]: Creating key for client.nfs.foo.1.0.smithi136.regjvc-rgw 2024-09-17T09:36:58.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:57 smithi078 ceph-mon[23689]: Deploying daemon nfs.foo.1.0.smithi136.regjvc on smithi136 2024-09-17T09:36:58.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:57 smithi078 ceph-mon[23689]: pgmap v237: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-09-17T09:36:58.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:58 smithi136 ceph-mon[25953]: from='client.14558 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:36:58.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:58 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:58.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:58 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:36:58.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:58 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:36:58.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:58 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:36:59.209 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:58 smithi078 ceph-mon[23689]: from='client.14558 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:36:59.210 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:58 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:36:59.210 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:58 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:36:59.210 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:58 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:36:59.210 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:58 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:37:00.153 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:59 smithi078 ceph-mon[23689]: pgmap v238: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-09-17T09:37:00.154 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:59 smithi078 ceph-mon[23689]: from='client.14562 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:37:00.154 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:36:59 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:37:00.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:59 smithi136 ceph-mon[25953]: pgmap v238: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-09-17T09:37:00.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:59 smithi136 ceph-mon[25953]: from='client.14562 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:37:00.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:36:59 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:37:01.187 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:00 smithi136 ceph-mon[25953]: from='client.14578 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:37:01.187 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:00 smithi136 ceph-mon[25953]: pgmap v239: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 1.6 KiB/s wr, 5 op/s 2024-09-17T09:37:01.343 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:00 smithi078 ceph-mon[23689]: from='client.14578 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:37:01.343 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:00 smithi078 ceph-mon[23689]: pgmap v239: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 1.6 KiB/s wr, 5 op/s 2024-09-17T09:37:02.315 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:02 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:37:02.315 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:02 smithi078 ceph-mon[23689]: from='client.14582 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:37:02.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:02 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:37:02.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:02 smithi136 ceph-mon[25953]: from='client.14582 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:37:03.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:03 smithi078 ceph-mon[23689]: pgmap v240: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-09-17T09:37:03.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:03 smithi078 ceph-mon[23689]: from='client.14586 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:37:03.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:03 smithi136 ceph-mon[25953]: pgmap v240: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-09-17T09:37:03.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:03 smithi136 ceph-mon[25953]: from='client.14586 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:37:03.880 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo ?:2049 2/2 2s ago 14s count:2 2024-09-17T09:37:04.563 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-17T09:37:04.575 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-17T09:37:04.575 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-09-17T09:37:04.601 INFO:teuthology.orchestra.run.smithi078.stderr:+ mkdir /mnt/foo 2024-09-17T09:37:04.630 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-09-17T09:37:04.691 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:04 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:37:04.692 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:04 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:37:04.692 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:04 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:37:04.692 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:04 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:37:04.692 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:04 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:37:04.692 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:04 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:37:04.692 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:04 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:37:04.692 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:04 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:37:04.692 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:04 smithi078 ceph-mon[23689]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:37:04.694 INFO:teuthology.orchestra.run.smithi078.stderr:++ hostname 2024-09-17T09:37:04.694 INFO:teuthology.orchestra.run.smithi078.stderr:+ mount -t nfs smithi078:/fake /mnt/foo -o sync 2024-09-17T09:37:04.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:04 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:37:04.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:04 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:37:04.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:04 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:37:04.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:04 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:37:04.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:04 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:37:04.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:04 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:37:04.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:04 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:37:04.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:04 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:37:04.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:04 smithi136 ceph-mon[25953]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:37:05.127 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-09-17T09:37:05.191 INFO:teuthology.orchestra.run.smithi078.stderr:+ echo test 2024-09-17T09:37:05.202 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-09-17T09:37:05.265 INFO:teuthology.orchestra.run.smithi078.stderr:+ sync 2024-09-17T09:37:05.523 INFO:teuthology.run_tasks:Running task parallel... 2024-09-17T09:37:05.535 INFO:teuthology.task.parallel:starting parallel... 2024-09-17T09:37:05.536 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-17T09:37:05.536 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-17T09:37:05.537 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-17T09:37:05.537 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -e sha1=e646f72961223a04d41eb40dad875d1628d78516 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-17T09:37:05.537 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-17T09:37:05.538 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-09-17T09:37:05.538 INFO:teuthology.task.exec:Executing custom commands... 2024-09-17T09:37:05.539 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-17T09:37:05.539 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-09-17T09:37:05.608 INFO:teuthology.orchestra.run.smithi078.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-17T09:37:05.609 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-17T09:37:05.609 INFO:teuthology.orchestra.run.smithi078.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-09-17T09:37:05.610 INFO:teuthology.orchestra.run.smithi078.stdout:3 of 5 processes prepared for launch 0 sec 2024-09-17T09:37:05.710 INFO:teuthology.orchestra.run.smithi078.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-17T09:37:05.710 INFO:teuthology.orchestra.run.smithi078.stdout:releasing clients 2024-09-17T09:37:05.720 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:05 smithi078 ceph-mon[23689]: pgmap v241: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-09-17T09:37:05.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:05 smithi136 ceph-mon[25953]: pgmap v241: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-09-17T09:37:06.710 INFO:teuthology.orchestra.run.smithi078.stdout: 5 296 54.50 MB/sec warmup 1 sec latency 41.493 ms 2024-09-17T09:37:07.710 INFO:teuthology.orchestra.run.smithi078.stdout: 5 617 51.57 MB/sec warmup 2 sec latency 13.062 ms 2024-09-17T09:37:07.763 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:07 smithi078 ceph-mon[23689]: pgmap v242: 97 pgs: 97 active+clean; 651 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.2 KiB/s wr, 6 op/s 2024-09-17T09:37:07.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:07 smithi136 ceph-mon[25953]: pgmap v242: 97 pgs: 97 active+clean; 651 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.2 KiB/s wr, 6 op/s 2024-09-17T09:37:08.710 INFO:teuthology.orchestra.run.smithi078.stdout: 5 1442 43.38 MB/sec warmup 3 sec latency 10.548 ms 2024-09-17T09:37:09.703 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -e sha1=e646f72961223a04d41eb40dad875d1628d78516 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-17T09:37:09.710 INFO:teuthology.orchestra.run.smithi078.stdout: 5 2572 36.80 MB/sec warmup 4 sec latency 12.296 ms 2024-09-17T09:37:09.733 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:09 smithi078 ceph-mon[23689]: pgmap v243: 97 pgs: 97 active+clean; 12 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 946 KiB/s wr, 21 op/s 2024-09-17T09:37:09.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:09 smithi136 ceph-mon[25953]: pgmap v243: 97 pgs: 97 active+clean; 12 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 946 KiB/s wr, 21 op/s 2024-09-17T09:37:10.710 INFO:teuthology.orchestra.run.smithi078.stdout: 5 3717 39.08 MB/sec warmup 5 sec latency 12.052 ms 2024-09-17T09:37:11.710 INFO:teuthology.orchestra.run.smithi078.stdout: 5 4384 33.83 MB/sec warmup 6 sec latency 130.363 ms 2024-09-17T09:37:11.779 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:11 smithi078 ceph-mon[23689]: pgmap v244: 97 pgs: 97 active+clean; 74 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 6.8 MiB/s wr, 273 op/s 2024-09-17T09:37:11.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:11 smithi136 ceph-mon[25953]: pgmap v244: 97 pgs: 97 active+clean; 74 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 6.8 MiB/s wr, 273 op/s 2024-09-17T09:37:12.710 INFO:teuthology.orchestra.run.smithi078.stdout: 5 5502 32.56 MB/sec warmup 7 sec latency 15.264 ms 2024-09-17T09:37:13.711 INFO:teuthology.orchestra.run.smithi078.stdout: 5 6331 31.55 MB/sec warmup 8 sec latency 27.268 ms 2024-09-17T09:37:13.790 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:13 smithi078 ceph-mon[23689]: pgmap v245: 97 pgs: 97 active+clean; 118 MiB data, 198 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 11 MiB/s wr, 506 op/s 2024-09-17T09:37:13.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:13 smithi136 ceph-mon[25953]: pgmap v245: 97 pgs: 97 active+clean; 118 MiB data, 198 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 11 MiB/s wr, 506 op/s 2024-09-17T09:37:14.711 INFO:teuthology.orchestra.run.smithi078.stdout: 5 7292 32.37 MB/sec warmup 9 sec latency 13.400 ms 2024-09-17T09:37:15.507 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -e sha1=e646f72961223a04d41eb40dad875d1628d78516 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-17T09:37:15.710 INFO:teuthology.orchestra.run.smithi078.stdout: 5 8069 30.19 MB/sec warmup 10 sec latency 15.616 ms 2024-09-17T09:37:15.799 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:15 smithi078 ceph-mon[23689]: pgmap v246: 97 pgs: 97 active+clean; 125 MiB data, 237 MiB used, 715 GiB / 715 GiB avail; 4.2 KiB/s rd, 13 MiB/s wr, 644 op/s 2024-09-17T09:37:15.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:15 smithi136 ceph-mon[25953]: pgmap v246: 97 pgs: 97 active+clean; 125 MiB data, 237 MiB used, 715 GiB / 715 GiB avail; 4.2 KiB/s rd, 13 MiB/s wr, 644 op/s 2024-09-17T09:37:16.711 INFO:teuthology.orchestra.run.smithi078.stdout: 5 8956 29.38 MB/sec warmup 11 sec latency 50.007 ms 2024-09-17T09:37:17.711 INFO:teuthology.orchestra.run.smithi078.stdout: 5 9420 27.39 MB/sec warmup 12 sec latency 50.318 ms 2024-09-17T09:37:17.735 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:17 smithi078 ceph-mon[23689]: pgmap v247: 97 pgs: 97 active+clean; 140 MiB data, 320 MiB used, 715 GiB / 715 GiB avail; 4.2 KiB/s rd, 16 MiB/s wr, 1.01k op/s 2024-09-17T09:37:17.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:17 smithi136 ceph-mon[25953]: pgmap v247: 97 pgs: 97 active+clean; 140 MiB data, 320 MiB used, 715 GiB / 715 GiB avail; 4.2 KiB/s rd, 16 MiB/s wr, 1.01k op/s 2024-09-17T09:37:18.711 INFO:teuthology.orchestra.run.smithi078.stdout: 5 9812 26.51 MB/sec warmup 13 sec latency 32.961 ms 2024-09-17T09:37:19.711 INFO:teuthology.orchestra.run.smithi078.stdout: 5 10178 25.80 MB/sec warmup 14 sec latency 49.876 ms 2024-09-17T09:37:19.851 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:19 smithi078 ceph-mon[23689]: pgmap v248: 97 pgs: 97 active+clean; 141 MiB data, 333 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 17 MiB/s wr, 1.06k op/s 2024-09-17T09:37:19.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:19 smithi136 ceph-mon[25953]: pgmap v248: 97 pgs: 97 active+clean; 141 MiB data, 333 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 17 MiB/s wr, 1.06k op/s 2024-09-17T09:37:20.708 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -e sha1=e646f72961223a04d41eb40dad875d1628d78516 -- bash -c 'ceph mgr module enable nfs --force' 2024-09-17T09:37:20.711 INFO:teuthology.orchestra.run.smithi078.stdout: 5 10530 24.89 MB/sec warmup 15 sec latency 134.114 ms 2024-09-17T09:37:21.678 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:21 smithi078 ceph-mon[23689]: pgmap v249: 97 pgs: 97 active+clean; 149 MiB data, 381 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 19 MiB/s wr, 1.31k op/s 2024-09-17T09:37:21.711 INFO:teuthology.orchestra.run.smithi078.stdout: 5 10867 24.25 MB/sec warmup 16 sec latency 44.330 ms 2024-09-17T09:37:21.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:21 smithi136 ceph-mon[25953]: pgmap v249: 97 pgs: 97 active+clean; 149 MiB data, 381 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 19 MiB/s wr, 1.31k op/s 2024-09-17T09:37:22.711 INFO:teuthology.orchestra.run.smithi078.stdout: 5 11107 23.03 MB/sec warmup 17 sec latency 50.765 ms 2024-09-17T09:37:23.711 INFO:teuthology.orchestra.run.smithi078.stdout: 5 11522 22.01 MB/sec warmup 18 sec latency 65.015 ms 2024-09-17T09:37:23.730 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:23 smithi078 ceph-mon[23689]: pgmap v250: 97 pgs: 97 active+clean; 146 MiB data, 412 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 14 MiB/s wr, 1.11k op/s 2024-09-17T09:37:23.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:23 smithi136 ceph-mon[25953]: pgmap v250: 97 pgs: 97 active+clean; 146 MiB data, 412 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 14 MiB/s wr, 1.11k op/s 2024-09-17T09:37:24.621 INFO:teuthology.orchestra.run.smithi078.stderr:module 'nfs' is already enabled 2024-09-17T09:37:24.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:24 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/1906470925' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-17T09:37:24.711 INFO:teuthology.orchestra.run.smithi078.stdout: 5 11726 20.96 MB/sec warmup 19 sec latency 121.857 ms 2024-09-17T09:37:24.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:24 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/1906470925' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-17T09:37:25.711 INFO:teuthology.orchestra.run.smithi078.stdout: 5 12221 20.73 MB/sec warmup 20 sec latency 47.374 ms 2024-09-17T09:37:25.778 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:25 smithi078 ceph-mon[23689]: pgmap v251: 97 pgs: 97 active+clean; 150 MiB data, 425 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 9.8 MiB/s wr, 943 op/s 2024-09-17T09:37:25.779 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:25 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/1906470925' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-17T09:37:25.779 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:25 smithi078 ceph-mon[23689]: mgrmap e21: smithi078.rsqofi(active, since 6m), standbys: smithi136.hicmlm 2024-09-17T09:37:25.922 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -e sha1=e646f72961223a04d41eb40dad875d1628d78516 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-17T09:37:25.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:25 smithi136 ceph-mon[25953]: pgmap v251: 97 pgs: 97 active+clean; 150 MiB data, 425 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 9.8 MiB/s wr, 943 op/s 2024-09-17T09:37:25.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:25 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/1906470925' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-17T09:37:25.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:25 smithi136 ceph-mon[25953]: mgrmap e21: smithi078.rsqofi(active, since 6m), standbys: smithi136.hicmlm 2024-09-17T09:37:26.711 INFO:teuthology.orchestra.run.smithi078.stdout: 5 12690 20.07 MB/sec warmup 21 sec latency 61.830 ms 2024-09-17T09:37:27.711 INFO:teuthology.orchestra.run.smithi078.stdout: 5 13083 19.46 MB/sec warmup 22 sec latency 40.462 ms 2024-09-17T09:37:27.830 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:27 smithi078 ceph-mon[23689]: pgmap v252: 97 pgs: 97 active+clean; 150 MiB data, 429 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 9.6 MiB/s wr, 903 op/s 2024-09-17T09:37:27.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:27 smithi136 ceph-mon[25953]: pgmap v252: 97 pgs: 97 active+clean; 150 MiB data, 429 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 9.6 MiB/s wr, 903 op/s 2024-09-17T09:37:28.711 INFO:teuthology.orchestra.run.smithi078.stdout: 5 13395 19.16 MB/sec warmup 23 sec latency 37.507 ms 2024-09-17T09:37:29.414 INFO:teuthology.orchestra.run.smithi078.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:e646f72961223a04d41eb40dad875d1628d78516 2024-09-17T09:37:29.618 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:29 smithi078 ceph-mon[23689]: pgmap v253: 97 pgs: 97 active+clean; 151 MiB data, 440 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 6.7 MiB/s wr, 654 op/s 2024-09-17T09:37:29.618 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:29 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:37:29.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:29 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:37:29.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:29 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:37:29.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:29 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:37:29.711 INFO:teuthology.orchestra.run.smithi078.stdout: 5 14032 19.46 MB/sec warmup 24 sec latency 51.483 ms 2024-09-17T09:37:29.780 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:29 smithi136 ceph-mon[25953]: pgmap v253: 97 pgs: 97 active+clean; 151 MiB data, 440 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 6.7 MiB/s wr, 654 op/s 2024-09-17T09:37:29.780 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:29 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:37:29.780 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:29 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:37:29.780 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:29 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:37:29.780 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:29 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:37:30.485 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-17T09:37:30.487 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-17T09:37:30.487 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -e sha1=e646f72961223a04d41eb40dad875d1628d78516 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-09-17T09:37:30.637 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:30 smithi078 ceph-mon[23689]: from='client.14608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:e646f72961223a04d41eb40dad875d1628d78516", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:37:30.637 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:30 smithi078 ceph-mon[23689]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:e646f72961223a04d41eb40dad875d1628d78516 2024-09-17T09:37:30.637 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:30 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:37:30.638 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:30 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:37:30.711 INFO:teuthology.orchestra.run.smithi078.stdout: 5 14433 19.34 MB/sec warmup 25 sec latency 39.128 ms 2024-09-17T09:37:30.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:30 smithi136 ceph-mon[25953]: from='client.14608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:e646f72961223a04d41eb40dad875d1628d78516", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:37:30.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:30 smithi136 ceph-mon[25953]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:e646f72961223a04d41eb40dad875d1628d78516 2024-09-17T09:37:30.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:30 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:37:30.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:30 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:37:31.643 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:31 smithi078 ceph-mon[23689]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:e646f72961223a04d41eb40dad875d1628d78516 2024-09-17T09:37:31.643 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:31 smithi078 ceph-mon[23689]: pgmap v254: 97 pgs: 97 active+clean; 155 MiB data, 465 MiB used, 715 GiB / 715 GiB avail; 6.8 KiB/s rd, 7.1 MiB/s wr, 708 op/s 2024-09-17T09:37:31.711 INFO:teuthology.orchestra.run.smithi078.stdout: 5 14816 18.78 MB/sec warmup 26 sec latency 32.924 ms 2024-09-17T09:37:31.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:31 smithi136 ceph-mon[25953]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:e646f72961223a04d41eb40dad875d1628d78516 2024-09-17T09:37:31.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:31 smithi136 ceph-mon[25953]: pgmap v254: 97 pgs: 97 active+clean; 155 MiB data, 465 MiB used, 715 GiB / 715 GiB avail; 6.8 KiB/s rd, 7.1 MiB/s wr, 708 op/s 2024-09-17T09:37:32.711 INFO:teuthology.orchestra.run.smithi078.stdout: 5 15481 18.65 MB/sec warmup 27 sec latency 41.780 ms 2024-09-17T09:37:33.659 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:33 smithi078 ceph-mon[23689]: pgmap v255: 97 pgs: 97 active+clean; 160 MiB data, 493 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.5 MiB/s wr, 476 op/s 2024-09-17T09:37:33.711 INFO:teuthology.orchestra.run.smithi078.stdout: 5 16295 18.46 MB/sec warmup 28 sec latency 42.483 ms 2024-09-17T09:37:33.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:33 smithi136 ceph-mon[25953]: pgmap v255: 97 pgs: 97 active+clean; 160 MiB data, 493 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.5 MiB/s wr, 476 op/s 2024-09-17T09:37:34.712 INFO:teuthology.orchestra.run.smithi078.stdout: 5 16883 18.45 MB/sec warmup 29 sec latency 33.206 ms 2024-09-17T09:37:35.675 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:35 smithi078 ceph-mon[23689]: pgmap v256: 97 pgs: 97 active+clean; 162 MiB data, 505 MiB used, 715 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.2 MiB/s wr, 499 op/s 2024-09-17T09:37:35.712 INFO:teuthology.orchestra.run.smithi078.stdout: 5 17704 18.98 MB/sec warmup 30 sec latency 31.313 ms 2024-09-17T09:37:35.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:35 smithi136 ceph-mon[25953]: pgmap v256: 97 pgs: 97 active+clean; 162 MiB data, 505 MiB used, 715 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.2 MiB/s wr, 499 op/s 2024-09-17T09:37:36.712 INFO:teuthology.orchestra.run.smithi078.stdout: 5 18137 18.78 MB/sec warmup 31 sec latency 50.592 ms 2024-09-17T09:37:37.411 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-17T09:37:37.699 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:37 smithi078 ceph-mon[23689]: pgmap v257: 97 pgs: 97 active+clean; 163 MiB data, 575 MiB used, 715 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.3 MiB/s wr, 570 op/s 2024-09-17T09:37:37.713 INFO:teuthology.orchestra.run.smithi078.stdout: 5 18504 18.28 MB/sec warmup 32 sec latency 41.805 ms 2024-09-17T09:37:37.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:37 smithi136 ceph-mon[25953]: pgmap v257: 97 pgs: 97 active+clean; 163 MiB data, 575 MiB used, 715 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.3 MiB/s wr, 570 op/s 2024-09-17T09:37:38.477 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:37:38.477 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (5m) 35s ago 7m 22.6M - ba2b418f427c 8143c3c75418 2024-09-17T09:37:38.477 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (7m) 35s ago 7m 6400k - 17.2.0 e1d6a67b021e 46d44333624b 2024-09-17T09:37:38.477 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi136 smithi136 running (6m) 37s ago 6m 6395k - 17.2.0 e1d6a67b021e 56ce035cb482 2024-09-17T09:37:38.477 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (5m) 35s ago 6m 53.8M - 8.3.5 dad864ee21e9 27395d6baf92 2024-09-17T09:37:38.477 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.jdrsup smithi078 running (65s) 35s ago 64s 13.5M - 17.2.0 e1d6a67b021e 8f834e15ba88 2024-09-17T09:37:38.477 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi136.smslff smithi136 running (62s) 37s ago 62s 13.4M - 17.2.0 e1d6a67b021e 0c20e2b328d3 2024-09-17T09:37:38.477 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rsqofi smithi078 *:9283 running (8m) 35s ago 8m 460M - 17.2.0 e1d6a67b021e f2012c57d1ac 2024-09-17T09:37:38.477 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi136.hicmlm smithi136 *:8443,9283 running (6m) 37s ago 6m 413M - 17.2.0 e1d6a67b021e b12edb7834e5 2024-09-17T09:37:38.478 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (8m) 35s ago 8m 47.3M 2048M 17.2.0 e1d6a67b021e 09299430e248 2024-09-17T09:37:38.478 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi136 smithi136 running (6m) 37s ago 6m 37.2M 2048M 17.2.0 e1d6a67b021e 1bfdfb0617bb 2024-09-17T09:37:38.478 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.tcwbvi smithi078 *:2049 running (45s) 35s ago 45s 49.0M - 4.0 e1d6a67b021e 30c87cb46555 2024-09-17T09:37:38.478 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi136.regjvc smithi136 *:2049 running (40s) 37s ago 39s 53.5M - 4.0 e1d6a67b021e b88e4dc24376 2024-09-17T09:37:38.478 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (6m) 35s ago 6m 18.4M - 1dbe0e931976 ca57ddb194b4 2024-09-17T09:37:38.478 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi136 smithi136 *:9100 running (6m) 37s ago 6m 17.6M - 1dbe0e931976 7205c98b1bae 2024-09-17T09:37:38.478 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (5m) 35s ago 5m 42.7M 3216M 17.2.0 e1d6a67b021e b4ca42f8477d 2024-09-17T09:37:38.478 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (5m) 35s ago 5m 43.0M 3216M 17.2.0 e1d6a67b021e 351873014de4 2024-09-17T09:37:38.478 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (4m) 35s ago 4m 40.3M 3216M 17.2.0 e1d6a67b021e 07eb71e4fa57 2024-09-17T09:37:38.478 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (4m) 35s ago 4m 40.4M 3216M 17.2.0 e1d6a67b021e 1b3a8be89881 2024-09-17T09:37:38.478 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi136 running (3m) 37s ago 3m 39.4M 3984M 17.2.0 e1d6a67b021e a0c5b4de2414 2024-09-17T09:37:38.478 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi136 running (3m) 37s ago 3m 38.2M 3984M 17.2.0 e1d6a67b021e 30a3049955f9 2024-09-17T09:37:38.478 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi136 running (3m) 37s ago 3m 39.4M 3984M 17.2.0 e1d6a67b021e 1698e1567c33 2024-09-17T09:37:38.478 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi136 running (2m) 37s ago 2m 40.9M 3984M 17.2.0 e1d6a67b021e 5d3b08cecd78 2024-09-17T09:37:38.479 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (6m) 35s ago 6m 64.9M - 514e6a882f6e 8405bb1fd6e6 2024-09-17T09:37:38.712 INFO:teuthology.orchestra.run.smithi078.stdout: 5 19119 18.25 MB/sec warmup 33 sec latency 23.554 ms 2024-09-17T09:37:38.713 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:38 smithi078 ceph-mon[23689]: from='client.14612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:37:38.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:38 smithi136 ceph-mon[25953]: from='client.14612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:37:39.116 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:37:39.116 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-17T09:37:39.116 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:37:39.116 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:37:39.117 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-17T09:37:39.117 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:37:39.117 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:37:39.117 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-17T09:37:39.117 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T09:37:39.117 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:37:39.117 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-17T09:37:39.117 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:37:39.117 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:37:39.117 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-17T09:37:39.117 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-17T09:37:39.117 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-17T09:37:39.117 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:37:39.675 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:37:39.675 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:e646f72961223a04d41eb40dad875d1628d78516", 2024-09-17T09:37:39.675 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-17T09:37:39.675 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-09-17T09:37:39.675 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "", 2024-09-17T09:37:39.675 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:e646f72961223a04d41eb40dad875d1628d78516 image" 2024-09-17T09:37:39.675 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:37:39.712 INFO:teuthology.orchestra.run.smithi078.stdout: 5 19880 18.04 MB/sec warmup 34 sec latency 24.853 ms 2024-09-17T09:37:39.717 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:39 smithi078 ceph-mon[23689]: from='client.24357 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:37:39.717 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:39 smithi078 ceph-mon[23689]: from='client.14620 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:37:39.718 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:39 smithi078 ceph-mon[23689]: pgmap v258: 97 pgs: 97 active+clean; 168 MiB data, 601 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 6.5 MiB/s wr, 636 op/s 2024-09-17T09:37:39.718 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:39 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/1019392922' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:37:39.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:39 smithi136 ceph-mon[25953]: from='client.24357 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:37:39.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:39 smithi136 ceph-mon[25953]: from='client.14620 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:37:39.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:39 smithi136 ceph-mon[25953]: pgmap v258: 97 pgs: 97 active+clean; 168 MiB data, 601 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 6.5 MiB/s wr, 636 op/s 2024-09-17T09:37:39.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:39 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/1019392922' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:37:40.308 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-17T09:37:40.712 INFO:teuthology.orchestra.run.smithi078.stdout: 5 20811 18.40 MB/sec warmup 35 sec latency 13.258 ms 2024-09-17T09:37:40.724 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:40 smithi078 ceph-mon[23689]: from='client.24369 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:37:40.724 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:40 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/3397105734' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:37:40.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:40 smithi136 ceph-mon[25953]: from='client.24369 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:37:40.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:40 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/3397105734' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:37:41.712 INFO:teuthology.orchestra.run.smithi078.stdout: 5 21579 18.76 MB/sec warmup 36 sec latency 34.822 ms 2024-09-17T09:37:41.810 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:41 smithi078 ceph-mon[23689]: pgmap v259: 97 pgs: 97 active+clean; 173 MiB data, 631 MiB used, 715 GiB / 715 GiB avail; 8.5 KiB/s rd, 7.8 MiB/s wr, 680 op/s 2024-09-17T09:37:41.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:41 smithi136 ceph-mon[25953]: pgmap v259: 97 pgs: 97 active+clean; 173 MiB data, 631 MiB used, 715 GiB / 715 GiB avail; 8.5 KiB/s rd, 7.8 MiB/s wr, 680 op/s 2024-09-17T09:37:42.712 INFO:teuthology.orchestra.run.smithi078.stdout: 5 21910 18.36 MB/sec warmup 37 sec latency 54.924 ms 2024-09-17T09:37:43.712 INFO:teuthology.orchestra.run.smithi078.stdout: 5 22371 18.05 MB/sec warmup 38 sec latency 53.741 ms 2024-09-17T09:37:43.842 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:43 smithi078 ceph-mon[23689]: pgmap v260: 97 pgs: 97 active+clean; 184 MiB data, 678 MiB used, 715 GiB / 715 GiB avail; 8.3 KiB/s rd, 8.7 MiB/s wr, 850 op/s 2024-09-17T09:37:43.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:43 smithi136 ceph-mon[25953]: pgmap v260: 97 pgs: 97 active+clean; 184 MiB data, 678 MiB used, 715 GiB / 715 GiB avail; 8.3 KiB/s rd, 8.7 MiB/s wr, 850 op/s 2024-09-17T09:37:44.712 INFO:teuthology.orchestra.run.smithi078.stdout: 5 23305 18.13 MB/sec warmup 39 sec latency 18.193 ms 2024-09-17T09:37:45.712 INFO:teuthology.orchestra.run.smithi078.stdout: 5 23939 18.09 MB/sec warmup 40 sec latency 55.130 ms 2024-09-17T09:37:45.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:45 smithi078 ceph-mon[23689]: pgmap v261: 97 pgs: 97 active+clean; 185 MiB data, 685 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 8.6 MiB/s wr, 897 op/s 2024-09-17T09:37:45.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:45 smithi136 ceph-mon[25953]: pgmap v261: 97 pgs: 97 active+clean; 185 MiB data, 685 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 8.6 MiB/s wr, 897 op/s 2024-09-17T09:37:46.712 INFO:teuthology.orchestra.run.smithi078.stdout: 5 24746 18.43 MB/sec warmup 41 sec latency 27.015 ms 2024-09-17T09:37:47.712 INFO:teuthology.orchestra.run.smithi078.stdout: 5 25421 18.44 MB/sec warmup 42 sec latency 19.330 ms 2024-09-17T09:37:47.862 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:47 smithi078 ceph-mon[23689]: pgmap v262: 97 pgs: 97 active+clean; 185 MiB data, 697 MiB used, 715 GiB / 715 GiB avail; 9.3 KiB/s rd, 9.6 MiB/s wr, 907 op/s 2024-09-17T09:37:47.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:47 smithi136 ceph-mon[25953]: pgmap v262: 97 pgs: 97 active+clean; 185 MiB data, 697 MiB used, 715 GiB / 715 GiB avail; 9.3 KiB/s rd, 9.6 MiB/s wr, 907 op/s 2024-09-17T09:37:48.712 INFO:teuthology.orchestra.run.smithi078.stdout: 5 26137 18.36 MB/sec warmup 43 sec latency 23.206 ms 2024-09-17T09:37:49.712 INFO:teuthology.orchestra.run.smithi078.stdout: 5 26926 18.28 MB/sec warmup 44 sec latency 25.081 ms 2024-09-17T09:37:49.872 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:49 smithi078 ceph-mon[23689]: pgmap v263: 97 pgs: 97 active+clean; 186 MiB data, 744 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.7 MiB/s wr, 822 op/s 2024-09-17T09:37:49.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:49 smithi136 ceph-mon[25953]: pgmap v263: 97 pgs: 97 active+clean; 186 MiB data, 744 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.7 MiB/s wr, 822 op/s 2024-09-17T09:37:50.712 INFO:teuthology.orchestra.run.smithi078.stdout: 5 27790 18.50 MB/sec warmup 45 sec latency 14.044 ms 2024-09-17T09:37:51.712 INFO:teuthology.orchestra.run.smithi078.stdout: 5 28707 18.87 MB/sec warmup 46 sec latency 25.269 ms 2024-09-17T09:37:51.880 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:51 smithi078 ceph-mon[23689]: pgmap v264: 97 pgs: 97 active+clean; 197 MiB data, 798 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 10 MiB/s wr, 913 op/s 2024-09-17T09:37:51.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:51 smithi136 ceph-mon[25953]: pgmap v264: 97 pgs: 97 active+clean; 197 MiB data, 798 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 10 MiB/s wr, 913 op/s 2024-09-17T09:37:52.712 INFO:teuthology.orchestra.run.smithi078.stdout: 5 29421 18.64 MB/sec warmup 47 sec latency 19.367 ms 2024-09-17T09:37:53.712 INFO:teuthology.orchestra.run.smithi078.stdout: 5 30166 18.62 MB/sec warmup 48 sec latency 65.400 ms 2024-09-17T09:37:53.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:53 smithi078 ceph-mon[23689]: pgmap v265: 97 pgs: 97 active+clean; 207 MiB data, 828 MiB used, 714 GiB / 715 GiB avail; 9.8 KiB/s rd, 11 MiB/s wr, 1.01k op/s 2024-09-17T09:37:53.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:53 smithi136 ceph-mon[25953]: pgmap v265: 97 pgs: 97 active+clean; 207 MiB data, 828 MiB used, 714 GiB / 715 GiB avail; 9.8 KiB/s rd, 11 MiB/s wr, 1.01k op/s 2024-09-17T09:37:54.712 INFO:teuthology.orchestra.run.smithi078.stdout: 5 30654 18.41 MB/sec warmup 49 sec latency 34.573 ms 2024-09-17T09:37:54.919 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:54 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.3", "id": [7, 4]}]: dispatch 2024-09-17T09:37:54.919 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:54 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-09-17T09:37:54.920 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:54 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-09-17T09:37:54.920 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:54 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-17T09:37:54.920 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:54 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:37:54.920 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:54 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/trash_purge_schedule"}]: dispatch 2024-09-17T09:37:54.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:54 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.3", "id": [7, 4]}]: dispatch 2024-09-17T09:37:54.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:54 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-09-17T09:37:54.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:54 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-09-17T09:37:54.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:54 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-17T09:37:54.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:54 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:37:54.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:54 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/trash_purge_schedule"}]: dispatch 2024-09-17T09:37:55.678 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:55 smithi078 ceph-mon[23689]: pgmap v266: 97 pgs: 97 active+clean; 209 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 9.4 MiB/s wr, 840 op/s 2024-09-17T09:37:55.678 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:55 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.3", "id": [7, 4]}]': finished 2024-09-17T09:37:55.681 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:55 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-09-17T09:37:55.682 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:55 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-09-17T09:37:55.682 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:55 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-09-17T09:37:55.682 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:55 smithi078 ceph-mon[23689]: osdmap e69: 8 total, 8 up, 8 in 2024-09-17T09:37:55.712 INFO:teuthology.orchestra.run.smithi078.stdout: 5 30948 18.22 MB/sec warmup 50 sec latency 60.639 ms 2024-09-17T09:37:55.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:55 smithi136 ceph-mon[25953]: pgmap v266: 97 pgs: 97 active+clean; 209 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 9.4 MiB/s wr, 840 op/s 2024-09-17T09:37:55.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:55 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.3", "id": [7, 4]}]': finished 2024-09-17T09:37:55.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:55 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-09-17T09:37:55.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:55 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-09-17T09:37:55.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:55 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-09-17T09:37:55.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:55 smithi136 ceph-mon[25953]: osdmap e69: 8 total, 8 up, 8 in 2024-09-17T09:37:56.690 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:56 smithi078 ceph-mon[23689]: osdmap e70: 8 total, 8 up, 8 in 2024-09-17T09:37:56.712 INFO:teuthology.orchestra.run.smithi078.stdout: 5 31405 18.24 MB/sec warmup 51 sec latency 53.612 ms 2024-09-17T09:37:56.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:56 smithi136 ceph-mon[25953]: osdmap e70: 8 total, 8 up, 8 in 2024-09-17T09:37:57.695 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:57 smithi078 ceph-mon[23689]: pgmap v269: 97 pgs: 2 peering, 95 active+clean; 213 MiB data, 899 MiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 13 MiB/s wr, 1.28k op/s; 0 B/s, 0 objects/s recovering 2024-09-17T09:37:57.695 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:57 smithi078 ceph-mon[23689]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-17T09:37:57.712 INFO:teuthology.orchestra.run.smithi078.stdout: 5 31907 18.29 MB/sec warmup 52 sec latency 33.792 ms 2024-09-17T09:37:57.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:57 smithi136 ceph-mon[25953]: pgmap v269: 97 pgs: 2 peering, 95 active+clean; 213 MiB data, 899 MiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 13 MiB/s wr, 1.28k op/s; 0 B/s, 0 objects/s recovering 2024-09-17T09:37:57.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:57 smithi136 ceph-mon[25953]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-17T09:37:58.712 INFO:teuthology.orchestra.run.smithi078.stdout: 5 32282 18.18 MB/sec warmup 53 sec latency 40.715 ms 2024-09-17T09:37:59.719 INFO:teuthology.orchestra.run.smithi078.stdout: 5 32684 17.91 MB/sec warmup 54 sec latency 72.830 ms 2024-09-17T09:37:59.775 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:37:59 smithi078 ceph-mon[23689]: pgmap v270: 97 pgs: 2 peering, 95 active+clean; 212 MiB data, 915 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.6 MiB/s wr, 946 op/s; 0 B/s, 0 objects/s recovering 2024-09-17T09:37:59.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:37:59 smithi136 ceph-mon[25953]: pgmap v270: 97 pgs: 2 peering, 95 active+clean; 212 MiB data, 915 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.6 MiB/s wr, 946 op/s; 0 B/s, 0 objects/s recovering 2024-09-17T09:38:00.719 INFO:teuthology.orchestra.run.smithi078.stdout: 5 33392 17.91 MB/sec warmup 55 sec latency 97.076 ms 2024-09-17T09:38:01.719 INFO:teuthology.orchestra.run.smithi078.stdout: 5 34292 17.84 MB/sec warmup 56 sec latency 30.528 ms 2024-09-17T09:38:01.794 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:01 smithi078 ceph-mon[23689]: pgmap v271: 97 pgs: 1 peering, 96 active+clean; 219 MiB data, 954 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 8.3 MiB/s wr, 751 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-09-17T09:38:01.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:01 smithi136 ceph-mon[25953]: pgmap v271: 97 pgs: 1 peering, 96 active+clean; 219 MiB data, 954 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 8.3 MiB/s wr, 751 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-09-17T09:38:02.719 INFO:teuthology.orchestra.run.smithi078.stdout: 5 34709 17.82 MB/sec warmup 57 sec latency 33.602 ms 2024-09-17T09:38:02.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:02 smithi078 ceph-mon[23689]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-17T09:38:02.808 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:02 smithi078 ceph-mon[23689]: Cluster is now healthy 2024-09-17T09:38:03.166 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:02 smithi136 ceph-mon[25953]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-17T09:38:03.166 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:02 smithi136 ceph-mon[25953]: Cluster is now healthy 2024-09-17T09:38:03.719 INFO:teuthology.orchestra.run.smithi078.stdout: 5 35324 17.96 MB/sec warmup 58 sec latency 43.529 ms 2024-09-17T09:38:03.834 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:03 smithi078 ceph-mon[23689]: pgmap v272: 97 pgs: 97 active+clean; 225 MiB data, 991 MiB used, 714 GiB / 715 GiB avail; 8.6 KiB/s rd, 8.8 MiB/s wr, 795 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-09-17T09:38:04.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:03 smithi136 ceph-mon[25953]: pgmap v272: 97 pgs: 97 active+clean; 225 MiB data, 991 MiB used, 714 GiB / 715 GiB avail; 8.6 KiB/s rd, 8.8 MiB/s wr, 795 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-09-17T09:38:04.719 INFO:teuthology.orchestra.run.smithi078.stdout: 5 35860 17.97 MB/sec warmup 59 sec latency 68.849 ms 2024-09-17T09:38:05.720 INFO:teuthology.orchestra.run.smithi078.stdout: 5 36113 17.71 MB/sec warmup 60 sec latency 55.571 ms 2024-09-17T09:38:05.872 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:05 smithi078 ceph-mon[23689]: pgmap v273: 97 pgs: 97 active+clean; 228 MiB data, 993 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.5 MiB/s wr, 755 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-09-17T09:38:06.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:05 smithi136 ceph-mon[25953]: pgmap v273: 97 pgs: 97 active+clean; 228 MiB data, 993 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.5 MiB/s wr, 755 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-09-17T09:38:06.719 INFO:teuthology.orchestra.run.smithi078.stdout: 5 36517 17.54 MB/sec warmup 61 sec latency 42.715 ms 2024-09-17T09:38:07.720 INFO:teuthology.orchestra.run.smithi078.stdout: 5 37209 17.54 MB/sec warmup 62 sec latency 60.716 ms 2024-09-17T09:38:07.920 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:07 smithi078 ceph-mon[23689]: pgmap v274: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 687 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-09-17T09:38:08.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:07 smithi136 ceph-mon[25953]: pgmap v274: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 687 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-09-17T09:38:08.720 INFO:teuthology.orchestra.run.smithi078.stdout: 5 37993 17.47 MB/sec warmup 63 sec latency 25.749 ms 2024-09-17T09:38:09.720 INFO:teuthology.orchestra.run.smithi078.stdout: 5 38768 17.73 MB/sec warmup 64 sec latency 29.334 ms 2024-09-17T09:38:09.935 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:09 smithi078 ceph-mon[23689]: pgmap v275: 97 pgs: 97 active+clean; 239 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 7.3 MiB/s wr, 669 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-09-17T09:38:09.936 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:09 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:38:09.936 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:09 smithi078 ceph-mon[23689]: Upgrade: Target is version 19.3.0-4963-ge646f729 (unknown) 2024-09-17T09:38:09.936 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:09 smithi078 ceph-mon[23689]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, digests ['quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407'] 2024-09-17T09:38:09.936 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:09 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:38:09.936 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:09 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:38:09.936 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:09 smithi078 ceph-mon[23689]: Upgrade: Need to upgrade myself (mgr.smithi078.rsqofi) 2024-09-17T09:38:09.936 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:09 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:38:10.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:09 smithi136 ceph-mon[25953]: pgmap v275: 97 pgs: 97 active+clean; 239 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 7.3 MiB/s wr, 669 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-09-17T09:38:10.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:09 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:38:10.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:09 smithi136 ceph-mon[25953]: Upgrade: Target is version 19.3.0-4963-ge646f729 (unknown) 2024-09-17T09:38:10.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:09 smithi136 ceph-mon[25953]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, digests ['quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407'] 2024-09-17T09:38:10.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:09 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:38:10.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:09 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:38:10.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:09 smithi136 ceph-mon[25953]: Upgrade: Need to upgrade myself (mgr.smithi078.rsqofi) 2024-09-17T09:38:10.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:09 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:38:10.720 INFO:teuthology.orchestra.run.smithi078.stdout: 5 39414 17.78 MB/sec warmup 65 sec latency 69.781 ms 2024-09-17T09:38:10.838 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-17T09:38:10.987 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:10 smithi078 ceph-mon[23689]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 on smithi136 2024-09-17T09:38:11.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:10 smithi136 ceph-mon[25953]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 on smithi136 2024-09-17T09:38:11.720 INFO:teuthology.orchestra.run.smithi078.stdout: 5 39878 17.57 MB/sec warmup 66 sec latency 34.858 ms 2024-09-17T09:38:11.882 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:38:11.882 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (6m) 68s ago 7m 22.6M - ba2b418f427c 8143c3c75418 2024-09-17T09:38:11.882 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (7m) 68s ago 7m 6400k - 17.2.0 e1d6a67b021e 46d44333624b 2024-09-17T09:38:11.882 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi136 smithi136 running (7m) 70s ago 7m 6395k - 17.2.0 e1d6a67b021e 56ce035cb482 2024-09-17T09:38:11.882 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (6m) 68s ago 7m 53.8M - 8.3.5 dad864ee21e9 27395d6baf92 2024-09-17T09:38:11.882 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.jdrsup smithi078 running (98s) 68s ago 98s 13.5M - 17.2.0 e1d6a67b021e 8f834e15ba88 2024-09-17T09:38:11.883 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi136.smslff smithi136 running (95s) 70s ago 95s 13.4M - 17.2.0 e1d6a67b021e 0c20e2b328d3 2024-09-17T09:38:11.883 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rsqofi smithi078 *:9283 running (8m) 68s ago 8m 460M - 17.2.0 e1d6a67b021e f2012c57d1ac 2024-09-17T09:38:11.883 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi136.hicmlm smithi136 *:8443,9283 running (7m) 70s ago 7m 413M - 17.2.0 e1d6a67b021e b12edb7834e5 2024-09-17T09:38:11.883 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (8m) 68s ago 9m 47.3M 2048M 17.2.0 e1d6a67b021e 09299430e248 2024-09-17T09:38:11.883 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi136 smithi136 running (6m) 70s ago 6m 37.2M 2048M 17.2.0 e1d6a67b021e 1bfdfb0617bb 2024-09-17T09:38:11.883 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.tcwbvi smithi078 *:2049 running (79s) 68s ago 78s 49.0M - 4.0 e1d6a67b021e 30c87cb46555 2024-09-17T09:38:11.883 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi136.regjvc smithi136 *:2049 running (73s) 70s ago 73s 53.5M - 4.0 e1d6a67b021e b88e4dc24376 2024-09-17T09:38:11.883 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (7m) 68s ago 7m 18.4M - 1dbe0e931976 ca57ddb194b4 2024-09-17T09:38:11.883 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi136 smithi136 *:9100 running (6m) 70s ago 6m 17.6M - 1dbe0e931976 7205c98b1bae 2024-09-17T09:38:11.883 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (6m) 68s ago 6m 42.7M 3216M 17.2.0 e1d6a67b021e b4ca42f8477d 2024-09-17T09:38:11.883 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (5m) 68s ago 5m 43.0M 3216M 17.2.0 e1d6a67b021e 351873014de4 2024-09-17T09:38:11.883 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (5m) 68s ago 5m 40.3M 3216M 17.2.0 e1d6a67b021e 07eb71e4fa57 2024-09-17T09:38:11.883 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (4m) 68s ago 4m 40.4M 3216M 17.2.0 e1d6a67b021e 1b3a8be89881 2024-09-17T09:38:11.884 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi136 running (4m) 70s ago 4m 39.4M 3984M 17.2.0 e1d6a67b021e a0c5b4de2414 2024-09-17T09:38:11.884 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi136 running (3m) 70s ago 3m 38.2M 3984M 17.2.0 e1d6a67b021e 30a3049955f9 2024-09-17T09:38:11.884 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi136 running (3m) 70s ago 3m 39.4M 3984M 17.2.0 e1d6a67b021e 1698e1567c33 2024-09-17T09:38:11.884 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi136 running (3m) 70s ago 3m 40.9M 3984M 17.2.0 e1d6a67b021e 5d3b08cecd78 2024-09-17T09:38:11.884 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (6m) 68s ago 6m 64.9M - 514e6a882f6e 8405bb1fd6e6 2024-09-17T09:38:11.993 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:11 smithi078 ceph-mon[23689]: pgmap v276: 97 pgs: 97 active+clean; 240 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 8.5 MiB/s wr, 755 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-09-17T09:38:11.993 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:11 smithi078 ceph-mon[23689]: from='client.24377 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:38:12.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:11 smithi136 ceph-mon[25953]: pgmap v276: 97 pgs: 97 active+clean; 240 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 8.5 MiB/s wr, 755 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-09-17T09:38:12.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:11 smithi136 ceph-mon[25953]: from='client.24377 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:38:12.483 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:38:12.483 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-17T09:38:12.483 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:38:12.483 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:38:12.484 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-17T09:38:12.484 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:38:12.484 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:38:12.484 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-17T09:38:12.484 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T09:38:12.484 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:38:12.484 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-17T09:38:12.484 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:38:12.484 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:38:12.484 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-17T09:38:12.484 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-17T09:38:12.484 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-17T09:38:12.484 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:38:12.720 INFO:teuthology.orchestra.run.smithi078.stdout: 5 40350 17.50 MB/sec warmup 67 sec latency 32.923 ms 2024-09-17T09:38:13.006 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:38:13.006 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407", 2024-09-17T09:38:13.006 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-17T09:38:13.006 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-09-17T09:38:13.007 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "0/23 daemons upgraded", 2024-09-17T09:38:13.007 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 image on host smithi136" 2024-09-17T09:38:13.007 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:38:13.014 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:12 smithi078 ceph-mon[23689]: from='client.24381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:38:13.014 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:12 smithi078 ceph-mon[23689]: from='client.24383 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:38:13.014 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:12 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/2966020681' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:38:13.165 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:12 smithi136 ceph-mon[25953]: from='client.24381 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:38:13.165 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:12 smithi136 ceph-mon[25953]: from='client.24383 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:38:13.165 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:12 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/2966020681' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:38:13.546 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-17T09:38:13.720 INFO:teuthology.orchestra.run.smithi078.stdout: 5 41002 17.43 MB/sec warmup 68 sec latency 32.583 ms 2024-09-17T09:38:13.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:13 smithi078 ceph-mon[23689]: pgmap v277: 97 pgs: 97 active+clean; 243 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 8.4 MiB/s wr, 770 op/s 2024-09-17T09:38:13.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:13 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/3884324146' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:38:14.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:13 smithi136 ceph-mon[25953]: pgmap v277: 97 pgs: 97 active+clean; 243 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 8.4 MiB/s wr, 770 op/s 2024-09-17T09:38:14.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:13 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/3884324146' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:38:14.720 INFO:teuthology.orchestra.run.smithi078.stdout: 5 41528 17.34 MB/sec warmup 69 sec latency 92.660 ms 2024-09-17T09:38:15.035 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:14 smithi078 ceph-mon[23689]: from='client.24389 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:38:15.126 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:14 smithi136 ceph-mon[25953]: from='client.24389 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:38:15.720 INFO:teuthology.orchestra.run.smithi078.stdout: 5 42303 17.53 MB/sec warmup 70 sec latency 33.588 ms 2024-09-17T09:38:15.950 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:15 smithi078 ceph-mon[23689]: pgmap v278: 97 pgs: 97 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.1 MiB/s wr, 786 op/s 2024-09-17T09:38:16.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:15 smithi136 ceph-mon[25953]: pgmap v278: 97 pgs: 97 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.1 MiB/s wr, 786 op/s 2024-09-17T09:38:16.720 INFO:teuthology.orchestra.run.smithi078.stdout: 5 43066 17.64 MB/sec warmup 71 sec latency 15.902 ms 2024-09-17T09:38:17.720 INFO:teuthology.orchestra.run.smithi078.stdout: 5 43588 17.51 MB/sec warmup 72 sec latency 75.835 ms 2024-09-17T09:38:17.961 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:17 smithi078 ceph-mon[23689]: pgmap v279: 97 pgs: 97 active+clean; 244 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.7 KiB/s rd, 8.4 MiB/s wr, 755 op/s 2024-09-17T09:38:18.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:17 smithi136 ceph-mon[25953]: pgmap v279: 97 pgs: 97 active+clean; 244 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.7 KiB/s rd, 8.4 MiB/s wr, 755 op/s 2024-09-17T09:38:18.720 INFO:teuthology.orchestra.run.smithi078.stdout: 5 43929 17.38 MB/sec warmup 73 sec latency 57.853 ms 2024-09-17T09:38:19.720 INFO:teuthology.orchestra.run.smithi078.stdout: 5 44622 17.32 MB/sec warmup 74 sec latency 39.756 ms 2024-09-17T09:38:19.735 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:19 smithi078 ceph-mon[23689]: pgmap v280: 97 pgs: 97 active+clean; 258 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.7 MiB/s wr, 813 op/s 2024-09-17T09:38:19.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:19 smithi136 ceph-mon[25953]: pgmap v280: 97 pgs: 97 active+clean; 258 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.7 MiB/s wr, 813 op/s 2024-09-17T09:38:20.720 INFO:teuthology.orchestra.run.smithi078.stdout: 5 45492 17.46 MB/sec warmup 75 sec latency 22.185 ms 2024-09-17T09:38:21.720 INFO:teuthology.orchestra.run.smithi078.stdout: 5 46511 17.73 MB/sec warmup 76 sec latency 46.195 ms 2024-09-17T09:38:21.767 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:21 smithi078 ceph-mon[23689]: pgmap v281: 97 pgs: 97 active+clean; 262 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 9.5 MiB/s wr, 909 op/s 2024-09-17T09:38:21.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:21 smithi136 ceph-mon[25953]: pgmap v281: 97 pgs: 97 active+clean; 262 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 9.5 MiB/s wr, 909 op/s 2024-09-17T09:38:22.720 INFO:teuthology.orchestra.run.smithi078.stdout: 5 47265 17.62 MB/sec warmup 77 sec latency 23.081 ms 2024-09-17T09:38:23.720 INFO:teuthology.orchestra.run.smithi078.stdout: 5 48162 17.70 MB/sec warmup 78 sec latency 33.063 ms 2024-09-17T09:38:23.775 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:23 smithi078 ceph-mon[23689]: pgmap v282: 97 pgs: 97 active+clean; 263 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 9.0 MiB/s wr, 904 op/s 2024-09-17T09:38:23.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:23 smithi136 ceph-mon[25953]: pgmap v282: 97 pgs: 97 active+clean; 263 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 9.0 MiB/s wr, 904 op/s 2024-09-17T09:38:24.720 INFO:teuthology.orchestra.run.smithi078.stdout: 5 48914 17.71 MB/sec warmup 79 sec latency 27.797 ms 2024-09-17T09:38:25.720 INFO:teuthology.orchestra.run.smithi078.stdout: 5 49529 17.79 MB/sec warmup 80 sec latency 43.462 ms 2024-09-17T09:38:25.818 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:25 smithi078 ceph-mon[23689]: pgmap v283: 97 pgs: 97 active+clean; 266 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.5 MiB/s wr, 829 op/s 2024-09-17T09:38:25.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:25 smithi136 ceph-mon[25953]: pgmap v283: 97 pgs: 97 active+clean; 266 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.5 MiB/s wr, 829 op/s 2024-09-17T09:38:26.720 INFO:teuthology.orchestra.run.smithi078.stdout: 5 50172 17.83 MB/sec warmup 81 sec latency 28.841 ms 2024-09-17T09:38:27.720 INFO:teuthology.orchestra.run.smithi078.stdout: 5 50911 17.76 MB/sec warmup 82 sec latency 54.622 ms 2024-09-17T09:38:27.836 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:27 smithi078 ceph-mon[23689]: pgmap v284: 97 pgs: 97 active+clean; 274 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 9.6 MiB/s wr, 822 op/s 2024-09-17T09:38:27.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:27 smithi136 ceph-mon[25953]: pgmap v284: 97 pgs: 97 active+clean; 274 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 9.6 MiB/s wr, 822 op/s 2024-09-17T09:38:28.720 INFO:teuthology.orchestra.run.smithi078.stdout: 5 51766 17.81 MB/sec warmup 83 sec latency 31.482 ms 2024-09-17T09:38:29.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:29 smithi078 ceph-mon[23689]: pgmap v285: 97 pgs: 97 active+clean; 285 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 11 MiB/s wr, 1.08k op/s 2024-09-17T09:38:29.720 INFO:teuthology.orchestra.run.smithi078.stdout: 5 52565 17.83 MB/sec warmup 84 sec latency 24.468 ms 2024-09-17T09:38:29.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:29 smithi136 ceph-mon[25953]: pgmap v285: 97 pgs: 97 active+clean; 285 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 11 MiB/s wr, 1.08k op/s 2024-09-17T09:38:30.720 INFO:teuthology.orchestra.run.smithi078.stdout: 5 53139 17.90 MB/sec warmup 85 sec latency 34.823 ms 2024-09-17T09:38:31.634 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:31 smithi078 ceph-mon[23689]: pgmap v286: 97 pgs: 97 active+clean; 280 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 901 op/s 2024-09-17T09:38:31.720 INFO:teuthology.orchestra.run.smithi078.stdout: 5 53634 17.87 MB/sec warmup 86 sec latency 32.351 ms 2024-09-17T09:38:31.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:31 smithi136 ceph-mon[25953]: pgmap v286: 97 pgs: 97 active+clean; 280 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 901 op/s 2024-09-17T09:38:32.720 INFO:teuthology.orchestra.run.smithi078.stdout: 5 53995 17.76 MB/sec warmup 87 sec latency 40.165 ms 2024-09-17T09:38:33.660 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:33 smithi078 ceph-mon[23689]: pgmap v287: 97 pgs: 97 active+clean; 284 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 9.8 MiB/s wr, 848 op/s 2024-09-17T09:38:33.720 INFO:teuthology.orchestra.run.smithi078.stdout: 5 54509 17.63 MB/sec warmup 88 sec latency 58.318 ms 2024-09-17T09:38:33.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:33 smithi136 ceph-mon[25953]: pgmap v287: 97 pgs: 97 active+clean; 284 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 9.8 MiB/s wr, 848 op/s 2024-09-17T09:38:34.721 INFO:teuthology.orchestra.run.smithi078.stdout: 5 54924 17.55 MB/sec warmup 89 sec latency 33.065 ms 2024-09-17T09:38:35.715 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:35 smithi078 ceph-mon[23689]: pgmap v288: 97 pgs: 97 active+clean; 290 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 9.8 MiB/s wr, 881 op/s 2024-09-17T09:38:35.720 INFO:teuthology.orchestra.run.smithi078.stdout: 5 55506 17.51 MB/sec warmup 90 sec latency 78.804 ms 2024-09-17T09:38:35.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:35 smithi136 ceph-mon[25953]: pgmap v288: 97 pgs: 97 active+clean; 290 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 9.8 MiB/s wr, 881 op/s 2024-09-17T09:38:36.720 INFO:teuthology.orchestra.run.smithi078.stdout: 5 56100 17.50 MB/sec warmup 91 sec latency 29.794 ms 2024-09-17T09:38:37.721 INFO:teuthology.orchestra.run.smithi078.stdout: 5 56573 17.54 MB/sec warmup 92 sec latency 35.041 ms 2024-09-17T09:38:37.735 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:37 smithi078 ceph-mon[23689]: pgmap v289: 97 pgs: 97 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 966 op/s 2024-09-17T09:38:37.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:37 smithi136 ceph-mon[25953]: pgmap v289: 97 pgs: 97 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 966 op/s 2024-09-17T09:38:38.721 INFO:teuthology.orchestra.run.smithi078.stdout: 5 57310 17.59 MB/sec warmup 93 sec latency 49.385 ms 2024-09-17T09:38:39.721 INFO:teuthology.orchestra.run.smithi078.stdout: 5 58015 17.53 MB/sec warmup 94 sec latency 21.706 ms 2024-09-17T09:38:39.746 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:39 smithi078 ceph-mon[23689]: pgmap v290: 97 pgs: 97 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.3 MiB/s wr, 913 op/s 2024-09-17T09:38:39.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:39 smithi136 ceph-mon[25953]: pgmap v290: 97 pgs: 97 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.3 MiB/s wr, 913 op/s 2024-09-17T09:38:40.721 INFO:teuthology.orchestra.run.smithi078.stdout: 5 58676 17.52 MB/sec warmup 95 sec latency 77.645 ms 2024-09-17T09:38:41.721 INFO:teuthology.orchestra.run.smithi078.stdout: 5 59596 17.58 MB/sec warmup 96 sec latency 24.233 ms 2024-09-17T09:38:41.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:41 smithi136 ceph-mon[25953]: pgmap v291: 97 pgs: 97 active+clean; 301 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 697 op/s 2024-09-17T09:38:41.796 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:41 smithi078 ceph-mon[23689]: pgmap v291: 97 pgs: 97 active+clean; 301 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 697 op/s 2024-09-17T09:38:42.721 INFO:teuthology.orchestra.run.smithi078.stdout: 5 60392 17.71 MB/sec warmup 97 sec latency 20.614 ms 2024-09-17T09:38:43.582 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:43 smithi078 ceph-mon[23689]: pgmap v292: 97 pgs: 97 active+clean; 306 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.5 MiB/s wr, 810 op/s 2024-09-17T09:38:43.721 INFO:teuthology.orchestra.run.smithi078.stdout: 5 61041 17.69 MB/sec warmup 98 sec latency 64.251 ms 2024-09-17T09:38:43.847 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:43 smithi136 ceph-mon[25953]: pgmap v292: 97 pgs: 97 active+clean; 306 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.5 MiB/s wr, 810 op/s 2024-09-17T09:38:44.117 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-17T09:38:44.721 INFO:teuthology.orchestra.run.smithi078.stdout: 5 61729 17.63 MB/sec warmup 99 sec latency 40.246 ms 2024-09-17T09:38:45.090 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:44 smithi078 ceph-mon[23689]: Upgrade: Updating mgr.smithi136.hicmlm 2024-09-17T09:38:45.091 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:44 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:38:45.091 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:44 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi136.hicmlm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T09:38:45.091 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:44 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T09:38:45.091 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:44 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:38:45.091 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:44 smithi078 ceph-mon[23689]: Deploying daemon mgr.smithi136.hicmlm on smithi136 2024-09-17T09:38:45.120 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:38:45.120 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (6m) 101s ago 8m 22.6M - ba2b418f427c 8143c3c75418 2024-09-17T09:38:45.121 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (8m) 101s ago 8m 6400k - 17.2.0 e1d6a67b021e 46d44333624b 2024-09-17T09:38:45.121 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi136 smithi136 running (7m) 103s ago 7m 6395k - 17.2.0 e1d6a67b021e 56ce035cb482 2024-09-17T09:38:45.121 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (6m) 101s ago 8m 53.8M - 8.3.5 dad864ee21e9 27395d6baf92 2024-09-17T09:38:45.121 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.jdrsup smithi078 running (2m) 101s ago 2m 13.5M - 17.2.0 e1d6a67b021e 8f834e15ba88 2024-09-17T09:38:45.121 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi136.smslff smithi136 running (2m) 103s ago 2m 13.4M - 17.2.0 e1d6a67b021e 0c20e2b328d3 2024-09-17T09:38:45.121 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rsqofi smithi078 *:9283 running (9m) 101s ago 9m 460M - 17.2.0 e1d6a67b021e f2012c57d1ac 2024-09-17T09:38:45.121 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi136.hicmlm smithi136 *:8443,9283 running (7m) 103s ago 7m 413M - 17.2.0 e1d6a67b021e b12edb7834e5 2024-09-17T09:38:45.121 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (9m) 101s ago 9m 47.3M 2048M 17.2.0 e1d6a67b021e 09299430e248 2024-09-17T09:38:45.121 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi136 smithi136 running (7m) 103s ago 7m 37.2M 2048M 17.2.0 e1d6a67b021e 1bfdfb0617bb 2024-09-17T09:38:45.121 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.tcwbvi smithi078 *:2049 running (112s) 101s ago 111s 49.0M - 4.0 e1d6a67b021e 30c87cb46555 2024-09-17T09:38:45.121 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi136.regjvc smithi136 *:2049 running (106s) 103s ago 106s 53.5M - 4.0 e1d6a67b021e b88e4dc24376 2024-09-17T09:38:45.122 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (7m) 101s ago 7m 18.4M - 1dbe0e931976 ca57ddb194b4 2024-09-17T09:38:45.122 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi136 smithi136 *:9100 running (7m) 103s ago 7m 17.6M - 1dbe0e931976 7205c98b1bae 2024-09-17T09:38:45.122 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (6m) 101s ago 6m 42.7M 3216M 17.2.0 e1d6a67b021e b4ca42f8477d 2024-09-17T09:38:45.122 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (6m) 101s ago 6m 43.0M 3216M 17.2.0 e1d6a67b021e 351873014de4 2024-09-17T09:38:45.122 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (5m) 101s ago 5m 40.3M 3216M 17.2.0 e1d6a67b021e 07eb71e4fa57 2024-09-17T09:38:45.122 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (5m) 101s ago 5m 40.4M 3216M 17.2.0 e1d6a67b021e 1b3a8be89881 2024-09-17T09:38:45.122 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi136 running (4m) 103s ago 4m 39.4M 3984M 17.2.0 e1d6a67b021e a0c5b4de2414 2024-09-17T09:38:45.122 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi136 running (4m) 103s ago 4m 38.2M 3984M 17.2.0 e1d6a67b021e 30a3049955f9 2024-09-17T09:38:45.122 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi136 running (4m) 103s ago 4m 39.4M 3984M 17.2.0 e1d6a67b021e 1698e1567c33 2024-09-17T09:38:45.122 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi136 running (3m) 103s ago 3m 40.9M 3984M 17.2.0 e1d6a67b021e 5d3b08cecd78 2024-09-17T09:38:45.122 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (7m) 101s ago 7m 64.9M - 514e6a882f6e 8405bb1fd6e6 2024-09-17T09:38:45.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:44 smithi136 ceph-mon[25953]: Upgrade: Updating mgr.smithi136.hicmlm 2024-09-17T09:38:45.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:44 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:38:45.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:44 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi136.hicmlm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T09:38:45.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:44 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T09:38:45.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:44 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:38:45.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:44 smithi136 ceph-mon[25953]: Deploying daemon mgr.smithi136.hicmlm on smithi136 2024-09-17T09:38:45.713 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:38:45.713 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-17T09:38:45.713 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:38:45.713 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:38:45.713 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-17T09:38:45.713 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:38:45.713 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:38:45.713 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-17T09:38:45.714 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T09:38:45.714 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:38:45.714 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-17T09:38:45.714 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:38:45.714 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:38:45.714 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-17T09:38:45.714 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-17T09:38:45.714 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-17T09:38:45.714 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:38:45.721 INFO:teuthology.orchestra.run.smithi078.stdout: 5 62440 17.65 MB/sec warmup 100 sec latency 32.744 ms 2024-09-17T09:38:46.095 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:45 smithi078 ceph-mon[23689]: from='client.14658 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:38:46.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:45 smithi078 ceph-mon[23689]: pgmap v293: 97 pgs: 97 active+clean; 311 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.3 MiB/s wr, 824 op/s 2024-09-17T09:38:46.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:45 smithi078 ceph-mon[23689]: from='client.14662 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:38:46.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:45 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/3936018896' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:38:46.168 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:45 smithi136 ceph-mon[25953]: from='client.14658 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:38:46.168 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:45 smithi136 ceph-mon[25953]: pgmap v293: 97 pgs: 97 active+clean; 311 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.3 MiB/s wr, 824 op/s 2024-09-17T09:38:46.168 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:45 smithi136 ceph-mon[25953]: from='client.14662 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:38:46.168 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:45 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/3936018896' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:38:46.212 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:38:46.212 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407", 2024-09-17T09:38:46.212 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-17T09:38:46.212 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-09-17T09:38:46.213 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "0/23 daemons upgraded", 2024-09-17T09:38:46.213 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading mgr daemons" 2024-09-17T09:38:46.213 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:38:46.721 INFO:teuthology.orchestra.run.smithi078.stdout: 5 63141 17.65 MB/sec warmup 101 sec latency 28.151 ms 2024-09-17T09:38:46.816 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-17T09:38:46.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:46 smithi136 ceph-mon[25953]: from='client.24405 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:38:46.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:46 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/4293264466' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:38:47.107 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:46 smithi078 ceph-mon[23689]: from='client.24405 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:38:47.107 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:46 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/4293264466' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:38:47.721 INFO:teuthology.orchestra.run.smithi078.stdout: 5 63705 17.72 MB/sec warmup 102 sec latency 49.506 ms 2024-09-17T09:38:48.112 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:47 smithi078 ceph-mon[23689]: from='client.14674 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:38:48.113 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:47 smithi078 ceph-mon[23689]: pgmap v294: 97 pgs: 97 active+clean; 310 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.8 MiB/s wr, 759 op/s 2024-09-17T09:38:48.113 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:47 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:38:48.113 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:47 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:38:48.113 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:47 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:38:48.113 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:47 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:38:48.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:47 smithi136 ceph-mon[25953]: from='client.14674 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:38:48.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:47 smithi136 ceph-mon[25953]: pgmap v294: 97 pgs: 97 active+clean; 310 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.8 MiB/s wr, 759 op/s 2024-09-17T09:38:48.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:47 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:38:48.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:47 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:38:48.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:47 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:38:48.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:47 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:38:48.721 INFO:teuthology.orchestra.run.smithi078.stdout: 5 64299 17.71 MB/sec warmup 103 sec latency 34.656 ms 2024-09-17T09:38:49.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:49 smithi078 ceph-mon[23689]: pgmap v295: 97 pgs: 97 active+clean; 321 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 9.8 MiB/s wr, 890 op/s 2024-09-17T09:38:49.721 INFO:teuthology.orchestra.run.smithi078.stdout: 5 64661 17.62 MB/sec warmup 104 sec latency 44.846 ms 2024-09-17T09:38:49.863 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:49 smithi136 ceph-mon[25953]: pgmap v295: 97 pgs: 97 active+clean; 321 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 9.8 MiB/s wr, 890 op/s 2024-09-17T09:38:50.721 INFO:teuthology.orchestra.run.smithi078.stdout: 5 65283 17.53 MB/sec warmup 105 sec latency 27.465 ms 2024-09-17T09:38:51.629 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:51 smithi078 ceph-mon[23689]: pgmap v296: 97 pgs: 97 active+clean; 325 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 956 op/s 2024-09-17T09:38:51.721 INFO:teuthology.orchestra.run.smithi078.stdout: 5 65953 17.55 MB/sec warmup 106 sec latency 29.284 ms 2024-09-17T09:38:51.862 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:51 smithi136 ceph-mon[25953]: pgmap v296: 97 pgs: 97 active+clean; 325 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 956 op/s 2024-09-17T09:38:52.721 INFO:teuthology.orchestra.run.smithi078.stdout: 5 66714 17.56 MB/sec warmup 107 sec latency 25.653 ms 2024-09-17T09:38:53.260 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:53 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:38:53.261 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:53 smithi078 ceph-mon[23689]: pgmap v297: 97 pgs: 97 active+clean; 335 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 1.01k op/s 2024-09-17T09:38:53.261 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:53 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:38:53.376 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:53 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:38:53.376 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:53 smithi136 ceph-mon[25953]: pgmap v297: 97 pgs: 97 active+clean; 335 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 1.01k op/s 2024-09-17T09:38:53.376 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:53 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:38:53.721 INFO:teuthology.orchestra.run.smithi078.stdout: 5 67298 17.62 MB/sec warmup 108 sec latency 30.341 ms 2024-09-17T09:38:54.721 INFO:teuthology.orchestra.run.smithi078.stdout: 5 67816 17.60 MB/sec warmup 109 sec latency 83.181 ms 2024-09-17T09:38:54.804 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:54 smithi078 ceph-mon[23689]: Standby manager daemon smithi136.hicmlm restarted 2024-09-17T09:38:54.804 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:54 smithi078 ceph-mon[23689]: Standby manager daemon smithi136.hicmlm started 2024-09-17T09:38:54.805 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:54 smithi078 ceph-mon[23689]: from='mgr.? 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi136.hicmlm/crt"}]: dispatch 2024-09-17T09:38:54.805 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:54 smithi078 ceph-mon[23689]: from='mgr.? 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T09:38:54.805 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:54 smithi078 ceph-mon[23689]: from='mgr.? 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi136.hicmlm/key"}]: dispatch 2024-09-17T09:38:54.805 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:54 smithi078 ceph-mon[23689]: from='mgr.? 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T09:38:54.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:54 smithi136 ceph-mon[25953]: Standby manager daemon smithi136.hicmlm restarted 2024-09-17T09:38:54.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:54 smithi136 ceph-mon[25953]: Standby manager daemon smithi136.hicmlm started 2024-09-17T09:38:54.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:54 smithi136 ceph-mon[25953]: from='mgr.? 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi136.hicmlm/crt"}]: dispatch 2024-09-17T09:38:54.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:54 smithi136 ceph-mon[25953]: from='mgr.? 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T09:38:54.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:54 smithi136 ceph-mon[25953]: from='mgr.? 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi136.hicmlm/key"}]: dispatch 2024-09-17T09:38:54.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:54 smithi136 ceph-mon[25953]: from='mgr.? 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T09:38:55.721 INFO:teuthology.orchestra.run.smithi078.stdout: 5 68197 17.53 MB/sec warmup 110 sec latency 28.670 ms 2024-09-17T09:38:55.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:55 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:38:55.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:55 smithi136 ceph-mon[25953]: pgmap v298: 97 pgs: 97 active+clean; 333 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 9.1 MiB/s wr, 876 op/s 2024-09-17T09:38:55.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:55 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/trash_purge_schedule"}]: dispatch 2024-09-17T09:38:55.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:55 smithi136 ceph-mon[25953]: mgrmap e22: smithi078.rsqofi(active, since 8m), standbys: smithi136.hicmlm 2024-09-17T09:38:55.826 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:55 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:38:55.826 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:55 smithi078 ceph-mon[23689]: pgmap v298: 97 pgs: 97 active+clean; 333 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 9.1 MiB/s wr, 876 op/s 2024-09-17T09:38:55.826 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:55 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/trash_purge_schedule"}]: dispatch 2024-09-17T09:38:55.826 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:55 smithi078 ceph-mon[23689]: mgrmap e22: smithi078.rsqofi(active, since 8m), standbys: smithi136.hicmlm 2024-09-17T09:38:56.721 INFO:teuthology.orchestra.run.smithi078.stdout: 5 68784 17.44 MB/sec warmup 111 sec latency 29.756 ms 2024-09-17T09:38:57.721 INFO:teuthology.orchestra.run.smithi078.stdout: 5 69412 17.44 MB/sec warmup 112 sec latency 30.646 ms 2024-09-17T09:38:57.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:57 smithi078 ceph-mon[23689]: pgmap v299: 97 pgs: 97 active+clean; 336 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 9.7 MiB/s wr, 916 op/s 2024-09-17T09:38:57.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:57 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:38:57.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:57 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:38:57.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:57 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:38:57.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:57 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:38:57.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:57 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mgr fail", "who": "smithi078.rsqofi"}]: dispatch 2024-09-17T09:38:57.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:57 smithi078 ceph-mon[23689]: osdmap e71: 8 total, 8 up, 8 in 2024-09-17T09:38:57.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:57 smithi136 ceph-mon[25953]: pgmap v299: 97 pgs: 97 active+clean; 336 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 9.7 MiB/s wr, 916 op/s 2024-09-17T09:38:57.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:57 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:38:57.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:57 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:38:57.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:57 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' 2024-09-17T09:38:57.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:57 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:38:57.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:57 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mgr fail", "who": "smithi078.rsqofi"}]: dispatch 2024-09-17T09:38:57.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:57 smithi136 ceph-mon[25953]: osdmap e71: 8 total, 8 up, 8 in 2024-09-17T09:38:58.721 INFO:teuthology.orchestra.run.smithi078.stdout: 5 69935 17.40 MB/sec warmup 113 sec latency 43.107 ms 2024-09-17T09:38:58.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:58 smithi136 ceph-mon[25953]: Upgrade: Need to upgrade myself (mgr.smithi078.rsqofi) 2024-09-17T09:38:58.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:58 smithi136 ceph-mon[25953]: Failing over to other MGR 2024-09-17T09:38:58.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:58 smithi136 ceph-mon[25953]: Standby manager daemon smithi078.rsqofi started 2024-09-17T09:38:58.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:58 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-17T09:38:58.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:58 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "mon metadata", "id": "smithi136"}]: dispatch 2024-09-17T09:38:58.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:58 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.jdrsup"}]: dispatch 2024-09-17T09:38:58.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:58 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi136.smslff"}]: dispatch 2024-09-17T09:38:58.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:58 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "mgr metadata", "who": "smithi136.hicmlm", "id": "smithi136.hicmlm"}]: dispatch 2024-09-17T09:38:58.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:58 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "mgr metadata", "who": "smithi078.rsqofi", "id": "smithi078.rsqofi"}]: dispatch 2024-09-17T09:38:58.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:58 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:38:58.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:58 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:38:58.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:58 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T09:38:58.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:58 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:38:58.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:58 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:38:58.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:58 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T09:38:58.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:58 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T09:38:58.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:58 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T09:38:58.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:58 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T09:38:58.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:58 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T09:38:58.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:58 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T09:38:58.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:58 smithi136 ceph-mon[25953]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "mgr fail", "who": "smithi078.rsqofi"}]': finished 2024-09-17T09:38:58.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:58 smithi136 ceph-mon[25953]: mgrmap e23: smithi136.hicmlm(active, starting, since 1.00531s), standbys: smithi078.rsqofi 2024-09-17T09:38:58.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:38:58 smithi136 ceph-mon[25953]: Manager daemon smithi136.hicmlm is now available 2024-09-17T09:38:58.865 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:58 smithi078 ceph-mon[23689]: Upgrade: Need to upgrade myself (mgr.smithi078.rsqofi) 2024-09-17T09:38:58.865 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:58 smithi078 ceph-mon[23689]: Failing over to other MGR 2024-09-17T09:38:58.865 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:58 smithi078 ceph-mon[23689]: Standby manager daemon smithi078.rsqofi started 2024-09-17T09:38:58.865 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:58 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-17T09:38:58.865 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:58 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "mon metadata", "id": "smithi136"}]: dispatch 2024-09-17T09:38:58.865 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:58 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.jdrsup"}]: dispatch 2024-09-17T09:38:58.866 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:58 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi136.smslff"}]: dispatch 2024-09-17T09:38:58.866 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:58 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "mgr metadata", "who": "smithi136.hicmlm", "id": "smithi136.hicmlm"}]: dispatch 2024-09-17T09:38:58.866 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:58 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "mgr metadata", "who": "smithi078.rsqofi", "id": "smithi078.rsqofi"}]: dispatch 2024-09-17T09:38:58.866 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:58 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:38:58.866 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:58 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:38:58.866 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:58 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T09:38:58.866 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:58 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:38:58.866 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:58 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:38:58.866 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:58 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T09:38:58.866 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:58 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T09:38:58.866 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:58 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T09:38:58.866 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:58 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T09:38:58.866 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:58 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T09:38:58.867 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:58 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T09:38:58.867 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:58 smithi078 ceph-mon[23689]: from='mgr.14202 172.21.15.78:0/1748617430' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "mgr fail", "who": "smithi078.rsqofi"}]': finished 2024-09-17T09:38:58.867 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:58 smithi078 ceph-mon[23689]: mgrmap e23: smithi136.hicmlm(active, starting, since 1.00531s), standbys: smithi078.rsqofi 2024-09-17T09:38:58.867 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:38:58 smithi078 ceph-mon[23689]: Manager daemon smithi136.hicmlm is now available 2024-09-17T09:38:59.721 INFO:teuthology.orchestra.run.smithi078.stdout: 5 70518 17.42 MB/sec warmup 114 sec latency 32.636 ms 2024-09-17T09:39:00.123 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:00 smithi078 ceph-mon[23689]: mgrmap e24: smithi136.hicmlm(active, since 2s), standbys: smithi078.rsqofi 2024-09-17T09:39:00.123 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:00 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:00.123 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:00 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:00.123 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:00 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:00.123 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:00 smithi078 ceph-mon[23689]: Checking for cert/key for grafana.smithi078 2024-09-17T09:39:00.123 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:00 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:00.123 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:00 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:00.124 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:00 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:39:00.124 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:00 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T09:39:00.124 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:00 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T09:39:00.124 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:00 smithi078 ceph-mon[23689]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T09:39:00.124 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:00 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi136.hicmlm/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:39:00.124 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:00 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi136.hicmlm/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:39:00.124 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:00 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi136.hicmlm/trash_purge_schedule"}]: dispatch 2024-09-17T09:39:00.124 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:00 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi136.hicmlm/trash_purge_schedule"}]: dispatch 2024-09-17T09:39:00.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:00 smithi136 ceph-mon[25953]: mgrmap e24: smithi136.hicmlm(active, since 2s), standbys: smithi078.rsqofi 2024-09-17T09:39:00.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:00 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:00.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:00 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:00.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:00 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:00.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:00 smithi136 ceph-mon[25953]: Checking for cert/key for grafana.smithi078 2024-09-17T09:39:00.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:00 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:00.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:00 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:00.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:00 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:39:00.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:00 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T09:39:00.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:00 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T09:39:00.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:00 smithi136 ceph-mon[25953]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T09:39:00.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:00 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi136.hicmlm/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:39:00.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:00 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi136.hicmlm/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:39:00.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:00 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi136.hicmlm/trash_purge_schedule"}]: dispatch 2024-09-17T09:39:00.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:00 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi136.hicmlm/trash_purge_schedule"}]: dispatch 2024-09-17T09:39:00.721 INFO:teuthology.orchestra.run.smithi078.stdout: 5 71151 17.46 MB/sec warmup 115 sec latency 62.581 ms 2024-09-17T09:39:01.129 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:01 smithi078 ceph-mon[23689]: Deploying cephadm binary to smithi136 2024-09-17T09:39:01.129 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:01 smithi078 ceph-mon[23689]: Deploying cephadm binary to smithi078 2024-09-17T09:39:01.129 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:01 smithi078 ceph-mon[23689]: [17/Sep/2024:09:39:00] ENGINE Bus STARTING 2024-09-17T09:39:01.129 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:01 smithi078 ceph-mon[23689]: pgmap v4: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-17T09:39:01.130 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:01 smithi078 ceph-mon[23689]: [17/Sep/2024:09:39:00] ENGINE Serving on https://172.21.15.136:7150 2024-09-17T09:39:01.130 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:01 smithi078 ceph-mon[23689]: [17/Sep/2024:09:39:00] ENGINE Error in HTTPServer.serve 2024-09-17T09:39:01.130 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Traceback (most recent call last): 2024-09-17T09:39:01.130 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T09:39:01.130 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self._connections.run(self.expiration_interval) 2024-09-17T09:39:01.130 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T09:39:01.130 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self._run(expiration_interval) 2024-09-17T09:39:01.130 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T09:39:01.131 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T09:39:01.131 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T09:39:01.131 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T09:39:01.131 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T09:39:01.131 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: s = self.context.wrap_socket( 2024-09-17T09:39:01.131 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T09:39:01.131 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: return self.sslsocket_class._create( 2024-09-17T09:39:01.131 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T09:39:01.131 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self.do_handshake() 2024-09-17T09:39:01.132 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T09:39:01.132 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self._sslobj.do_handshake() 2024-09-17T09:39:01.132 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T09:39:01.132 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:01 smithi078 ceph-mon[23689]: [17/Sep/2024:09:39:00] ENGINE Serving on http://172.21.15.136:8765 2024-09-17T09:39:01.132 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:01 smithi078 ceph-mon[23689]: [17/Sep/2024:09:39:00] ENGINE Bus STARTED 2024-09-17T09:39:01.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:01 smithi136 ceph-mon[25953]: Deploying cephadm binary to smithi136 2024-09-17T09:39:01.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:01 smithi136 ceph-mon[25953]: Deploying cephadm binary to smithi078 2024-09-17T09:39:01.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:01 smithi136 ceph-mon[25953]: [17/Sep/2024:09:39:00] ENGINE Bus STARTING 2024-09-17T09:39:01.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:01 smithi136 ceph-mon[25953]: pgmap v4: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-17T09:39:01.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:01 smithi136 ceph-mon[25953]: [17/Sep/2024:09:39:00] ENGINE Serving on https://172.21.15.136:7150 2024-09-17T09:39:01.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:01 smithi136 ceph-mon[25953]: [17/Sep/2024:09:39:00] ENGINE Error in HTTPServer.serve 2024-09-17T09:39:01.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: Traceback (most recent call last): 2024-09-17T09:39:01.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T09:39:01.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: self._connections.run(self.expiration_interval) 2024-09-17T09:39:01.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T09:39:01.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: self._run(expiration_interval) 2024-09-17T09:39:01.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T09:39:01.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T09:39:01.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T09:39:01.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T09:39:01.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T09:39:01.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: s = self.context.wrap_socket( 2024-09-17T09:39:01.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T09:39:01.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: return self.sslsocket_class._create( 2024-09-17T09:39:01.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T09:39:01.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: self.do_handshake() 2024-09-17T09:39:01.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T09:39:01.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: self._sslobj.do_handshake() 2024-09-17T09:39:01.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T09:39:01.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:01 smithi136 ceph-mon[25953]: [17/Sep/2024:09:39:00] ENGINE Serving on http://172.21.15.136:8765 2024-09-17T09:39:01.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:01 smithi136 ceph-mon[25953]: [17/Sep/2024:09:39:00] ENGINE Bus STARTED 2024-09-17T09:39:01.721 INFO:teuthology.orchestra.run.smithi078.stdout: 5 71684 17.42 MB/sec warmup 116 sec latency 28.630 ms 2024-09-17T09:39:02.135 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:02 smithi078 ceph-mon[23689]: mgrmap e25: smithi136.hicmlm(active, since 4s), standbys: smithi078.rsqofi 2024-09-17T09:39:02.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:02 smithi136 ceph-mon[25953]: mgrmap e25: smithi136.hicmlm(active, since 4s), standbys: smithi078.rsqofi 2024-09-17T09:39:02.722 INFO:teuthology.orchestra.run.smithi078.stdout: 5 72209 17.35 MB/sec warmup 117 sec latency 68.543 ms 2024-09-17T09:39:03.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:03 smithi078 ceph-mon[23689]: pgmap v5: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-17T09:39:03.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:03 smithi136 ceph-mon[25953]: pgmap v5: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-17T09:39:03.721 INFO:teuthology.orchestra.run.smithi078.stdout: 5 72683 17.28 MB/sec warmup 118 sec latency 78.041 ms 2024-09-17T09:39:04.164 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:04 smithi078 ceph-mon[23689]: mgrmap e26: smithi136.hicmlm(active, since 6s), standbys: smithi078.rsqofi 2024-09-17T09:39:04.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:04 smithi136 ceph-mon[25953]: mgrmap e26: smithi136.hicmlm(active, since 6s), standbys: smithi078.rsqofi 2024-09-17T09:39:04.721 INFO:teuthology.orchestra.run.smithi078.stdout: 5 73180 17.26 MB/sec warmup 119 sec latency 75.080 ms 2024-09-17T09:39:05.676 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:05 smithi078 ceph-mon[23689]: pgmap v6: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-17T09:39:05.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:05 smithi136 ceph-mon[25953]: pgmap v6: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-17T09:39:06.722 INFO:teuthology.orchestra.run.smithi078.stdout: 5 74168 11.00 MB/sec execute 1 sec latency 82.018 ms 2024-09-17T09:39:07.713 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:07 smithi078 ceph-mon[23689]: pgmap v7: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 5.9 MiB/s wr, 566 op/s 2024-09-17T09:39:07.713 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:07 smithi078 ceph-mon[23689]: Standby manager daemon smithi078.rsqofi restarted 2024-09-17T09:39:07.713 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:07 smithi078 ceph-mon[23689]: Standby manager daemon smithi078.rsqofi started 2024-09-17T09:39:07.714 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:07 smithi078 ceph-mon[23689]: from='mgr.? 172.21.15.78:0/2923090807' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.rsqofi/crt"}]: dispatch 2024-09-17T09:39:07.714 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:07 smithi078 ceph-mon[23689]: from='mgr.? 172.21.15.78:0/2923090807' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T09:39:07.714 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:07 smithi078 ceph-mon[23689]: from='mgr.? 172.21.15.78:0/2923090807' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.rsqofi/key"}]: dispatch 2024-09-17T09:39:07.714 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:07 smithi078 ceph-mon[23689]: from='mgr.? 172.21.15.78:0/2923090807' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T09:39:07.730 INFO:teuthology.orchestra.run.smithi078.stdout: 5 74753 15.48 MB/sec execute 2 sec latency 59.739 ms 2024-09-17T09:39:07.882 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:07 smithi136 ceph-mon[25953]: pgmap v7: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 5.9 MiB/s wr, 566 op/s 2024-09-17T09:39:07.883 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:07 smithi136 ceph-mon[25953]: Standby manager daemon smithi078.rsqofi restarted 2024-09-17T09:39:07.883 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:07 smithi136 ceph-mon[25953]: Standby manager daemon smithi078.rsqofi started 2024-09-17T09:39:07.883 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:07 smithi136 ceph-mon[25953]: from='mgr.? 172.21.15.78:0/2923090807' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.rsqofi/crt"}]: dispatch 2024-09-17T09:39:07.883 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:07 smithi136 ceph-mon[25953]: from='mgr.? 172.21.15.78:0/2923090807' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T09:39:07.883 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:07 smithi136 ceph-mon[25953]: from='mgr.? 172.21.15.78:0/2923090807' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.rsqofi/key"}]: dispatch 2024-09-17T09:39:07.883 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:07 smithi136 ceph-mon[25953]: from='mgr.? 172.21.15.78:0/2923090807' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T09:39:08.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:08 smithi078 ceph-mon[23689]: mgrmap e27: smithi136.hicmlm(active, since 10s), standbys: smithi078.rsqofi 2024-09-17T09:39:08.731 INFO:teuthology.orchestra.run.smithi078.stdout: 5 75330 15.74 MB/sec execute 3 sec latency 32.013 ms 2024-09-17T09:39:08.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:08 smithi136 ceph-mon[25953]: mgrmap e27: smithi136.hicmlm(active, since 10s), standbys: smithi078.rsqofi 2024-09-17T09:39:09.731 INFO:teuthology.orchestra.run.smithi078.stdout: 5 75735 13.34 MB/sec execute 4 sec latency 34.228 ms 2024-09-17T09:39:09.740 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:09 smithi078 ceph-mon[23689]: pgmap v8: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.6 MiB/s wr, 440 op/s 2024-09-17T09:39:09.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:09 smithi136 ceph-mon[25953]: pgmap v8: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.6 MiB/s wr, 440 op/s 2024-09-17T09:39:10.731 INFO:teuthology.orchestra.run.smithi078.stdout: 5 76135 11.97 MB/sec execute 5 sec latency 39.298 ms 2024-09-17T09:39:11.731 INFO:teuthology.orchestra.run.smithi078.stdout: 5 76505 11.41 MB/sec execute 6 sec latency 35.092 ms 2024-09-17T09:39:11.777 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:11 smithi078 ceph-mon[23689]: pgmap v9: 97 pgs: 97 active+clean; 338 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 7.5 MiB/s wr, 686 op/s 2024-09-17T09:39:11.777 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:11 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:11.778 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:11 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:11.778 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:11 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:11.778 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:11 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:11.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:11 smithi136 ceph-mon[25953]: pgmap v9: 97 pgs: 97 active+clean; 338 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 7.5 MiB/s wr, 686 op/s 2024-09-17T09:39:11.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:11 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:11.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:11 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:11.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:11 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:11.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:11 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:12.731 INFO:teuthology.orchestra.run.smithi078.stdout: 5 77137 12.36 MB/sec execute 7 sec latency 51.443 ms 2024-09-17T09:39:12.785 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:12 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:12.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:12 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:12.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:12 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:12.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:12 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:12.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:12 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:12.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:12 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:12.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:12 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:12.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:12 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:12.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:12 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:12.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:12 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:12.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:12 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:12.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:12 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:12.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:12 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:12.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:12 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:12.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:12 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:12.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:12 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:12.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:12 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:12.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:12 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:12.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:12 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:12.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:12 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:12.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:12 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:12.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:12 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:12.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:12 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:12.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:12 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:12.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:12 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:12.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:12 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:13.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:13 smithi136 ceph-mon[25953]: Adjusting osd_memory_target on smithi078 to 1936M 2024-09-17T09:39:13.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:13 smithi136 ceph-mon[25953]: pgmap v10: 97 pgs: 97 active+clean; 338 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 6.8 MiB/s wr, 628 op/s 2024-09-17T09:39:13.731 INFO:teuthology.orchestra.run.smithi078.stdout: 5 77951 14.74 MB/sec execute 8 sec latency 18.436 ms 2024-09-17T09:39:13.783 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:13 smithi078 ceph-mon[23689]: Adjusting osd_memory_target on smithi078 to 1936M 2024-09-17T09:39:13.783 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:13 smithi078 ceph-mon[23689]: pgmap v10: 97 pgs: 97 active+clean; 338 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 6.8 MiB/s wr, 628 op/s 2024-09-17T09:39:14.735 INFO:teuthology.orchestra.run.smithi078.stdout: 5 78723 15.33 MB/sec execute 9 sec latency 41.210 ms 2024-09-17T09:39:14.789 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:14 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T09:39:14.886 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:14 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T09:39:15.735 INFO:teuthology.orchestra.run.smithi078.stdout: 5 79055 14.42 MB/sec execute 10 sec latency 47.636 ms 2024-09-17T09:39:15.800 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:15 smithi078 ceph-mon[23689]: pgmap v11: 97 pgs: 97 active+clean; 339 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 7.2 MiB/s wr, 678 op/s 2024-09-17T09:39:15.800 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:15 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:15.801 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:15 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:15.801 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:15 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:15.801 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:15 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:15.801 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:15 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:15.801 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:15 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:15.801 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:15 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:15.801 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:15 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:15.801 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:15 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:15.801 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:15 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:15.801 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:15 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:15.801 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:15 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:39:15.801 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:15 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:39:15.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:15 smithi136 ceph-mon[25953]: pgmap v11: 97 pgs: 97 active+clean; 339 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 7.2 MiB/s wr, 678 op/s 2024-09-17T09:39:15.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:15 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:15.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:15 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:15.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:15 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:15.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:15 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:15.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:15 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:15.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:15 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:15.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:15 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:15.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:15 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:15.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:15 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:15.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:15 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T09:39:15.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:15 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:15.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:15 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:39:15.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:15 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:39:16.737 INFO:teuthology.orchestra.run.smithi078.stdout: 5 79535 14.03 MB/sec execute 11 sec latency 42.809 ms 2024-09-17T09:39:16.823 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:16 smithi078 ceph-mon[23689]: Adjusting osd_memory_target on smithi136 to 2704M 2024-09-17T09:39:16.823 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:16 smithi078 ceph-mon[23689]: Updating smithi078:/etc/ceph/ceph.conf 2024-09-17T09:39:16.823 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:16 smithi078 ceph-mon[23689]: Updating smithi136:/etc/ceph/ceph.conf 2024-09-17T09:39:16.823 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:16 smithi078 ceph-mon[23689]: Updating smithi136:/var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T09:39:16.823 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:16 smithi078 ceph-mon[23689]: Updating smithi078:/var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T09:39:16.823 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:16 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:16.823 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:16 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:16.823 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:16 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:16.823 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:16 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:16.823 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:16 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:16.823 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:16 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:16.824 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:16 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:16.824 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:16 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:16.824 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:16 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:16.824 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:16 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:16.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:16 smithi136 ceph-mon[25953]: Adjusting osd_memory_target on smithi136 to 2704M 2024-09-17T09:39:16.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:16 smithi136 ceph-mon[25953]: Updating smithi078:/etc/ceph/ceph.conf 2024-09-17T09:39:16.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:16 smithi136 ceph-mon[25953]: Updating smithi136:/etc/ceph/ceph.conf 2024-09-17T09:39:16.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:16 smithi136 ceph-mon[25953]: Updating smithi136:/var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T09:39:16.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:16 smithi136 ceph-mon[25953]: Updating smithi078:/var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T09:39:16.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:16 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:16.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:16 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:16.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:16 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:16.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:16 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:16.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:16 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:16.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:16 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:16.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:16 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:16.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:16 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:16.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:16 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:16.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:16 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:17.364 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-17T09:39:17.738 INFO:teuthology.orchestra.run.smithi078.stdout: 5 79936 13.37 MB/sec execute 12 sec latency 80.716 ms 2024-09-17T09:39:17.850 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:17 smithi078 ceph-mon[23689]: Updating smithi136:/etc/ceph/ceph.client.admin.keyring 2024-09-17T09:39:17.850 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:17 smithi078 ceph-mon[23689]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-09-17T09:39:17.850 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:17 smithi078 ceph-mon[23689]: Updating smithi136:/var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T09:39:17.850 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:17 smithi078 ceph-mon[23689]: pgmap v12: 97 pgs: 97 active+clean; 334 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 10 MiB/s wr, 952 op/s 2024-09-17T09:39:17.850 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:17 smithi078 ceph-mon[23689]: Updating smithi078:/var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T09:39:17.850 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:17 smithi078 ceph-mon[23689]: Reconfiguring alertmanager.smithi078 (dependencies changed)... 2024-09-17T09:39:17.850 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:17 smithi078 ceph-mon[23689]: Deploying daemon alertmanager.smithi078 on smithi078 2024-09-17T09:39:17.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:17 smithi136 ceph-mon[25953]: Updating smithi136:/etc/ceph/ceph.client.admin.keyring 2024-09-17T09:39:17.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:17 smithi136 ceph-mon[25953]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-09-17T09:39:17.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:17 smithi136 ceph-mon[25953]: Updating smithi136:/var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T09:39:17.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:17 smithi136 ceph-mon[25953]: pgmap v12: 97 pgs: 97 active+clean; 334 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 10 MiB/s wr, 952 op/s 2024-09-17T09:39:17.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:17 smithi136 ceph-mon[25953]: Updating smithi078:/var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T09:39:17.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:17 smithi136 ceph-mon[25953]: Reconfiguring alertmanager.smithi078 (dependencies changed)... 2024-09-17T09:39:17.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:17 smithi136 ceph-mon[25953]: Deploying daemon alertmanager.smithi078 on smithi078 2024-09-17T09:39:18.340 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:39:18.341 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (7m) 7s ago 8m 26.3M - 0.23.0 ba2b418f427c 8143c3c75418 2024-09-17T09:39:18.341 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (8m) 7s ago 8m 6400k - 17.2.0 e1d6a67b021e 46d44333624b 2024-09-17T09:39:18.341 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi136 smithi136 running (8m) 7s ago 8m 6395k - 17.2.0 e1d6a67b021e 56ce035cb482 2024-09-17T09:39:18.341 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (7m) 7s ago 8m 56.3M - dad864ee21e9 27395d6baf92 2024-09-17T09:39:18.341 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.jdrsup smithi078 running (2m) 7s ago 2m 37.0M - 17.2.0 e1d6a67b021e 8f834e15ba88 2024-09-17T09:39:18.341 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi136.smslff smithi136 running (2m) 7s ago 2m 14.4M - 17.2.0 e1d6a67b021e 0c20e2b328d3 2024-09-17T09:39:18.341 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rsqofi smithi078 *:9283 running (10m) 7s ago 10m 166M - 17.2.0 e1d6a67b021e f2012c57d1ac 2024-09-17T09:39:18.341 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi136.hicmlm smithi136 *:8443,9283 running (31s) 7s ago 8m 497M - 19.3.0-4963-ge646f729 f271290a0950 bbd092a7e4ca 2024-09-17T09:39:18.341 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (10m) 7s ago 10m 54.8M 2048M 17.2.0 e1d6a67b021e 09299430e248 2024-09-17T09:39:18.341 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi136 smithi136 running (8m) 7s ago 8m 38.3M 2048M 17.2.0 e1d6a67b021e 1bfdfb0617bb 2024-09-17T09:39:18.341 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.tcwbvi smithi078 *:2049 running (2m) 7s ago 2m 262M - 4.0 e1d6a67b021e 30c87cb46555 2024-09-17T09:39:18.341 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi136.regjvc smithi136 *:2049 running (2m) 7s ago 2m 77.5M - 4.0 e1d6a67b021e b88e4dc24376 2024-09-17T09:39:18.342 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (8m) 7s ago 8m 18.2M - 1.3.1 1dbe0e931976 ca57ddb194b4 2024-09-17T09:39:18.342 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi136 smithi136 *:9100 running (7m) 7s ago 7m 18.0M - 1.3.1 1dbe0e931976 7205c98b1bae 2024-09-17T09:39:18.342 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (7m) 7s ago 7m 292M 1936M 17.2.0 e1d6a67b021e b4ca42f8477d 2024-09-17T09:39:18.342 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (6m) 7s ago 6m 236M 1936M 17.2.0 e1d6a67b021e 351873014de4 2024-09-17T09:39:18.342 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (6m) 7s ago 6m 176M 1936M 17.2.0 e1d6a67b021e 07eb71e4fa57 2024-09-17T09:39:18.342 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (5m) 7s ago 5m 250M 1936M 17.2.0 e1d6a67b021e 1b3a8be89881 2024-09-17T09:39:18.342 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi136 running (5m) 7s ago 5m 263M 2704M 17.2.0 e1d6a67b021e a0c5b4de2414 2024-09-17T09:39:18.342 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi136 running (5m) 7s ago 5m 297M 2704M 17.2.0 e1d6a67b021e 30a3049955f9 2024-09-17T09:39:18.342 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi136 running (4m) 7s ago 4m 267M 2704M 17.2.0 e1d6a67b021e 1698e1567c33 2024-09-17T09:39:18.342 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi136 running (4m) 7s ago 4m 373M 2704M 17.2.0 e1d6a67b021e 5d3b08cecd78 2024-09-17T09:39:18.342 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (7m) 7s ago 7m 64.9M - 2.33.4 514e6a882f6e 8405bb1fd6e6 2024-09-17T09:39:18.738 INFO:teuthology.orchestra.run.smithi078.stdout: 5 80489 13.79 MB/sec execute 13 sec latency 47.340 ms 2024-09-17T09:39:18.879 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:18 smithi078 ceph-mon[23689]: from='client.24449 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:39:18.943 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:39:18.944 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-17T09:39:18.944 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:39:18.944 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:39:18.944 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-17T09:39:18.944 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T09:39:18.944 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 1 2024-09-17T09:39:18.944 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:39:18.944 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-17T09:39:18.944 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T09:39:18.944 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:39:18.945 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-17T09:39:18.945 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:39:18.945 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:39:18.945 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-17T09:39:18.945 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-17T09:39:18.945 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 1 2024-09-17T09:39:18.945 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-17T09:39:18.945 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:39:18.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:18 smithi136 ceph-mon[25953]: from='client.24449 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:39:19.404 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:39:19.404 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407", 2024-09-17T09:39:19.404 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-17T09:39:19.405 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T09:39:19.405 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-09-17T09:39:19.405 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "1/23 daemons upgraded", 2024-09-17T09:39:19.405 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "", 2024-09-17T09:39:19.405 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-17T09:39:19.405 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:39:19.738 INFO:teuthology.orchestra.run.smithi078.stdout: 5 81040 13.95 MB/sec execute 14 sec latency 41.081 ms 2024-09-17T09:39:19.884 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:19 smithi078 ceph-mon[23689]: from='client.14708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:39:19.885 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:19 smithi078 ceph-mon[23689]: pgmap v13: 97 pgs: 97 active+clean; 334 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.9 MiB/s wr, 623 op/s 2024-09-17T09:39:19.885 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:19 smithi078 ceph-mon[23689]: from='client.14712 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:39:19.885 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:19 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/525229078' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:39:19.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:19 smithi136 ceph-mon[25953]: from='client.14708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:39:19.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:19 smithi136 ceph-mon[25953]: pgmap v13: 97 pgs: 97 active+clean; 334 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.9 MiB/s wr, 623 op/s 2024-09-17T09:39:19.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:19 smithi136 ceph-mon[25953]: from='client.14712 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:39:19.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:19 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/525229078' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:39:20.009 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-17T09:39:20.684 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:20 smithi078 ceph-mon[23689]: from='client.14720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:39:20.684 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:20 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/2956823573' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:39:20.738 INFO:teuthology.orchestra.run.smithi078.stdout: 5 81355 13.80 MB/sec execute 15 sec latency 55.558 ms 2024-09-17T09:39:20.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:20 smithi136 ceph-mon[25953]: from='client.14720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:39:20.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:20 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/2956823573' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:39:21.710 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:21 smithi078 ceph-mon[23689]: pgmap v14: 97 pgs: 97 active+clean; 327 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.5 MiB/s wr, 914 op/s 2024-09-17T09:39:21.738 INFO:teuthology.orchestra.run.smithi078.stdout: 5 81813 13.80 MB/sec execute 16 sec latency 57.117 ms 2024-09-17T09:39:21.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:21 smithi136 ceph-mon[25953]: pgmap v14: 97 pgs: 97 active+clean; 327 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.5 MiB/s wr, 914 op/s 2024-09-17T09:39:22.738 INFO:teuthology.orchestra.run.smithi078.stdout: 5 82190 13.61 MB/sec execute 17 sec latency 53.089 ms 2024-09-17T09:39:23.738 INFO:teuthology.orchestra.run.smithi078.stdout: 5 82539 13.30 MB/sec execute 18 sec latency 46.415 ms 2024-09-17T09:39:23.764 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:23 smithi078 ceph-mon[23689]: pgmap v15: 97 pgs: 97 active+clean; 327 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.1 MiB/s wr, 615 op/s 2024-09-17T09:39:23.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:23 smithi136 ceph-mon[25953]: pgmap v15: 97 pgs: 97 active+clean; 327 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.1 MiB/s wr, 615 op/s 2024-09-17T09:39:24.738 INFO:teuthology.orchestra.run.smithi078.stdout: 5 82962 12.92 MB/sec execute 19 sec latency 63.615 ms 2024-09-17T09:39:25.738 INFO:teuthology.orchestra.run.smithi078.stdout: 5 83389 12.68 MB/sec execute 20 sec latency 68.437 ms 2024-09-17T09:39:25.817 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:25 smithi078 ceph-mon[23689]: pgmap v16: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.3 MiB/s wr, 629 op/s 2024-09-17T09:39:25.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:25 smithi136 ceph-mon[25953]: pgmap v16: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.3 MiB/s wr, 629 op/s 2024-09-17T09:39:26.741 INFO:teuthology.orchestra.run.smithi078.stdout: 5 83985 13.04 MB/sec execute 21 sec latency 44.712 ms 2024-09-17T09:39:27.741 INFO:teuthology.orchestra.run.smithi078.stdout: 5 84549 13.15 MB/sec execute 22 sec latency 40.355 ms 2024-09-17T09:39:27.858 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:27 smithi078 ceph-mon[23689]: pgmap v17: 97 pgs: 97 active+clean; 330 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.5 MiB/s wr, 809 op/s 2024-09-17T09:39:27.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:27 smithi136 ceph-mon[25953]: pgmap v17: 97 pgs: 97 active+clean; 330 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.5 MiB/s wr, 809 op/s 2024-09-17T09:39:28.741 INFO:teuthology.orchestra.run.smithi078.stdout: 5 85095 13.48 MB/sec execute 23 sec latency 46.917 ms 2024-09-17T09:39:29.741 INFO:teuthology.orchestra.run.smithi078.stdout: 5 85638 13.57 MB/sec execute 24 sec latency 22.379 ms 2024-09-17T09:39:29.874 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:29 smithi078 ceph-mon[23689]: pgmap v18: 97 pgs: 97 active+clean; 330 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.4 MiB/s wr, 534 op/s 2024-09-17T09:39:29.874 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:29 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T09:39:29.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:29 smithi136 ceph-mon[25953]: pgmap v18: 97 pgs: 97 active+clean; 330 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.4 MiB/s wr, 534 op/s 2024-09-17T09:39:29.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:29 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T09:39:30.741 INFO:teuthology.orchestra.run.smithi078.stdout: 5 86002 13.37 MB/sec execute 25 sec latency 49.139 ms 2024-09-17T09:39:31.741 INFO:teuthology.orchestra.run.smithi078.stdout: 5 86332 13.07 MB/sec execute 26 sec latency 83.443 ms 2024-09-17T09:39:31.933 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:31 smithi078 ceph-mon[23689]: pgmap v19: 97 pgs: 97 active+clean; 334 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 831 op/s 2024-09-17T09:39:31.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:31 smithi136 ceph-mon[25953]: pgmap v19: 97 pgs: 97 active+clean; 334 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 831 op/s 2024-09-17T09:39:32.741 INFO:teuthology.orchestra.run.smithi078.stdout: 5 86777 12.83 MB/sec execute 27 sec latency 40.625 ms 2024-09-17T09:39:33.741 INFO:teuthology.orchestra.run.smithi078.stdout: 5 87203 12.78 MB/sec execute 28 sec latency 61.513 ms 2024-09-17T09:39:33.761 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:33 smithi078 ceph-mon[23689]: pgmap v20: 97 pgs: 97 active+clean; 334 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.9 MiB/s wr, 540 op/s 2024-09-17T09:39:33.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:33 smithi136 ceph-mon[25953]: pgmap v20: 97 pgs: 97 active+clean; 334 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.9 MiB/s wr, 540 op/s 2024-09-17T09:39:34.741 INFO:teuthology.orchestra.run.smithi078.stdout: 5 87725 12.95 MB/sec execute 29 sec latency 32.953 ms 2024-09-17T09:39:35.742 INFO:teuthology.orchestra.run.smithi078.stdout: 5 88321 13.08 MB/sec execute 30 sec latency 75.002 ms 2024-09-17T09:39:35.784 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:35 smithi078 ceph-mon[23689]: pgmap v21: 97 pgs: 97 active+clean; 332 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.3 MiB/s wr, 568 op/s 2024-09-17T09:39:35.784 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:35 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:35.784 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:35 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:35.784 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:35 smithi078 ceph-mon[23689]: Reconfiguring grafana.smithi078 (dependencies changed)... 2024-09-17T09:39:35.784 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:35 smithi078 ceph-mon[23689]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-17T09:39:35.784 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:35 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:35.784 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:35 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:35.784 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:35 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T09:39:35.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:35 smithi136 ceph-mon[25953]: pgmap v21: 97 pgs: 97 active+clean; 332 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.3 MiB/s wr, 568 op/s 2024-09-17T09:39:35.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:35 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:35.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:35 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:35.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:35 smithi136 ceph-mon[25953]: Reconfiguring grafana.smithi078 (dependencies changed)... 2024-09-17T09:39:35.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:35 smithi136 ceph-mon[25953]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-17T09:39:35.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:35 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:35.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:35 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:35.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:35 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T09:39:36.742 INFO:teuthology.orchestra.run.smithi078.stdout: 5 88805 13.21 MB/sec execute 31 sec latency 40.697 ms 2024-09-17T09:39:36.790 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:36 smithi078 ceph-mon[23689]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T09:39:36.790 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:36 smithi078 ceph-mon[23689]: Reconfiguring daemon grafana.smithi078 on smithi078 2024-09-17T09:39:36.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:36 smithi136 ceph-mon[25953]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T09:39:36.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:36 smithi136 ceph-mon[25953]: Reconfiguring daemon grafana.smithi078 on smithi078 2024-09-17T09:39:37.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:37 smithi078 ceph-mon[23689]: pgmap v22: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 797 op/s 2024-09-17T09:39:37.742 INFO:teuthology.orchestra.run.smithi078.stdout: 5 89246 13.20 MB/sec execute 32 sec latency 48.070 ms 2024-09-17T09:39:37.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:37 smithi136 ceph-mon[25953]: pgmap v22: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 797 op/s 2024-09-17T09:39:38.742 INFO:teuthology.orchestra.run.smithi078.stdout: 5 89575 13.02 MB/sec execute 33 sec latency 39.115 ms 2024-09-17T09:39:39.742 INFO:teuthology.orchestra.run.smithi078.stdout: 5 89992 12.82 MB/sec execute 34 sec latency 41.561 ms 2024-09-17T09:39:39.762 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:39 smithi078 ceph-mon[23689]: pgmap v23: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.6 MiB/s wr, 567 op/s 2024-09-17T09:39:39.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:39 smithi136 ceph-mon[25953]: pgmap v23: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.6 MiB/s wr, 567 op/s 2024-09-17T09:39:40.742 INFO:teuthology.orchestra.run.smithi078.stdout: 5 90507 12.74 MB/sec execute 35 sec latency 38.857 ms 2024-09-17T09:39:41.742 INFO:teuthology.orchestra.run.smithi078.stdout: 5 91047 12.90 MB/sec execute 36 sec latency 46.901 ms 2024-09-17T09:39:41.814 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:41 smithi078 ceph-mon[23689]: pgmap v24: 97 pgs: 97 active+clean; 327 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 836 op/s 2024-09-17T09:39:41.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:41 smithi136 ceph-mon[25953]: pgmap v24: 97 pgs: 97 active+clean; 327 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 836 op/s 2024-09-17T09:39:42.742 INFO:teuthology.orchestra.run.smithi078.stdout: 5 91622 12.96 MB/sec execute 37 sec latency 38.732 ms 2024-09-17T09:39:43.742 INFO:teuthology.orchestra.run.smithi078.stdout: 5 92035 13.01 MB/sec execute 38 sec latency 43.107 ms 2024-09-17T09:39:43.823 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:43 smithi078 ceph-mon[23689]: pgmap v25: 97 pgs: 97 active+clean; 327 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.5 MiB/s wr, 539 op/s 2024-09-17T09:39:43.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:43 smithi136 ceph-mon[25953]: pgmap v25: 97 pgs: 97 active+clean; 327 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.5 MiB/s wr, 539 op/s 2024-09-17T09:39:44.742 INFO:teuthology.orchestra.run.smithi078.stdout: 5 92441 13.02 MB/sec execute 39 sec latency 97.590 ms 2024-09-17T09:39:45.086 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:44 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:45.086 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:44 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:45.086 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:44 smithi078 ceph-mon[23689]: Reconfiguring node-exporter.smithi078 (dependencies changed)... 2024-09-17T09:39:45.086 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:44 smithi078 ceph-mon[23689]: Deploying daemon node-exporter.smithi078 on smithi078 2024-09-17T09:39:45.086 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:44 smithi078 ceph-mon[23689]: pgmap v26: 97 pgs: 97 active+clean; 329 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 7.0 MiB/s wr, 707 op/s 2024-09-17T09:39:45.086 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:44 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T09:39:45.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:44 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:45.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:44 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:45.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:44 smithi136 ceph-mon[25953]: Reconfiguring node-exporter.smithi078 (dependencies changed)... 2024-09-17T09:39:45.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:44 smithi136 ceph-mon[25953]: Deploying daemon node-exporter.smithi078 on smithi078 2024-09-17T09:39:45.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:44 smithi136 ceph-mon[25953]: pgmap v26: 97 pgs: 97 active+clean; 329 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 7.0 MiB/s wr, 707 op/s 2024-09-17T09:39:45.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:44 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T09:39:45.742 INFO:teuthology.orchestra.run.smithi078.stdout: 5 92925 13.00 MB/sec execute 40 sec latency 58.193 ms 2024-09-17T09:39:46.742 INFO:teuthology.orchestra.run.smithi078.stdout: 5 93221 12.85 MB/sec execute 41 sec latency 105.190 ms 2024-09-17T09:39:47.412 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:47 smithi078 ceph-mon[23689]: pgmap v27: 97 pgs: 97 active+clean; 329 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 804 op/s 2024-09-17T09:39:47.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:47 smithi136 ceph-mon[25953]: pgmap v27: 97 pgs: 97 active+clean; 329 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 804 op/s 2024-09-17T09:39:47.752 INFO:teuthology.orchestra.run.smithi078.stdout: 5 93671 12.76 MB/sec execute 42 sec latency 44.378 ms 2024-09-17T09:39:48.755 INFO:teuthology.orchestra.run.smithi078.stdout: 5 94313 12.80 MB/sec execute 43 sec latency 28.435 ms 2024-09-17T09:39:49.675 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:49 smithi078 ceph-mon[23689]: pgmap v28: 97 pgs: 97 active+clean; 329 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.0 MiB/s wr, 562 op/s 2024-09-17T09:39:49.756 INFO:teuthology.orchestra.run.smithi078.stdout: 5 94996 12.98 MB/sec execute 44 sec latency 31.561 ms 2024-09-17T09:39:49.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:49 smithi136 ceph-mon[25953]: pgmap v28: 97 pgs: 97 active+clean; 329 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.0 MiB/s wr, 562 op/s 2024-09-17T09:39:50.574 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-17T09:39:50.756 INFO:teuthology.orchestra.run.smithi078.stdout: 5 95459 12.99 MB/sec execute 45 sec latency 31.996 ms 2024-09-17T09:39:51.648 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:39:51.648 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 starting - - - - 2024-09-17T09:39:51.649 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (9m) 40s ago 9m 6400k - 17.2.0 e1d6a67b021e 46d44333624b 2024-09-17T09:39:51.649 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi136 smithi136 running (8m) 40s ago 8m 6395k - 17.2.0 e1d6a67b021e 56ce035cb482 2024-09-17T09:39:51.649 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 starting - - - - 2024-09-17T09:39:51.649 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.jdrsup smithi078 running (3m) 40s ago 3m 37.0M - 17.2.0 e1d6a67b021e 8f834e15ba88 2024-09-17T09:39:51.649 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi136.smslff smithi136 running (3m) 40s ago 3m 14.4M - 17.2.0 e1d6a67b021e 0c20e2b328d3 2024-09-17T09:39:51.649 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rsqofi smithi078 *:9283 running (10m) 40s ago 10m 166M - 17.2.0 e1d6a67b021e f2012c57d1ac 2024-09-17T09:39:51.649 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi136.hicmlm smithi136 *:8443,9283 running (64s) 40s ago 8m 497M - 19.3.0-4963-ge646f729 f271290a0950 bbd092a7e4ca 2024-09-17T09:39:51.649 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (10m) 40s ago 10m 54.8M 2048M 17.2.0 e1d6a67b021e 09299430e248 2024-09-17T09:39:51.649 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi136 smithi136 running (8m) 40s ago 8m 38.3M 2048M 17.2.0 e1d6a67b021e 1bfdfb0617bb 2024-09-17T09:39:51.649 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.tcwbvi smithi078 *:2049 running (2m) 40s ago 2m 262M - 4.0 e1d6a67b021e 30c87cb46555 2024-09-17T09:39:51.649 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi136.regjvc smithi136 *:2049 running (2m) 40s ago 2m 77.5M - 4.0 e1d6a67b021e b88e4dc24376 2024-09-17T09:39:51.649 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (9m) 40s ago 9m 18.2M - 1.3.1 1dbe0e931976 ca57ddb194b4 2024-09-17T09:39:51.649 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi136 smithi136 *:9100 running (8m) 40s ago 8m 18.0M - 1.3.1 1dbe0e931976 7205c98b1bae 2024-09-17T09:39:51.650 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (7m) 40s ago 7m 292M 1936M 17.2.0 e1d6a67b021e b4ca42f8477d 2024-09-17T09:39:51.650 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (7m) 40s ago 7m 236M 1936M 17.2.0 e1d6a67b021e 351873014de4 2024-09-17T09:39:51.650 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (6m) 40s ago 6m 176M 1936M 17.2.0 e1d6a67b021e 07eb71e4fa57 2024-09-17T09:39:51.650 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (6m) 40s ago 6m 250M 1936M 17.2.0 e1d6a67b021e 1b3a8be89881 2024-09-17T09:39:51.650 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi136 running (6m) 40s ago 6m 263M 2704M 17.2.0 e1d6a67b021e a0c5b4de2414 2024-09-17T09:39:51.650 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi136 running (5m) 40s ago 5m 297M 2704M 17.2.0 e1d6a67b021e 30a3049955f9 2024-09-17T09:39:51.650 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi136 running (5m) 40s ago 5m 267M 2704M 17.2.0 e1d6a67b021e 1698e1567c33 2024-09-17T09:39:51.650 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi136 running (4m) 40s ago 4m 373M 2704M 17.2.0 e1d6a67b021e 5d3b08cecd78 2024-09-17T09:39:51.650 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (8m) 40s ago 8m 64.9M - 2.33.4 514e6a882f6e 8405bb1fd6e6 2024-09-17T09:39:51.705 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:51 smithi078 ceph-mon[23689]: pgmap v29: 97 pgs: 97 active+clean; 326 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.7 MiB/s wr, 843 op/s 2024-09-17T09:39:51.705 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:51 smithi078 ceph-mon[23689]: from='client.14728 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:39:51.759 INFO:teuthology.orchestra.run.smithi078.stdout: 5 95968 13.10 MB/sec execute 46 sec latency 37.487 ms 2024-09-17T09:39:51.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:51 smithi136 ceph-mon[25953]: pgmap v29: 97 pgs: 97 active+clean; 326 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.7 MiB/s wr, 843 op/s 2024-09-17T09:39:51.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:51 smithi136 ceph-mon[25953]: from='client.14728 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:39:52.324 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:39:52.324 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-17T09:39:52.324 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:39:52.324 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:39:52.324 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-17T09:39:52.325 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T09:39:52.325 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 1 2024-09-17T09:39:52.325 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:39:52.325 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-17T09:39:52.325 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T09:39:52.325 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:39:52.325 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-17T09:39:52.325 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:39:52.325 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:39:52.325 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-17T09:39:52.325 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-17T09:39:52.325 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 1 2024-09-17T09:39:52.326 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-17T09:39:52.326 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:39:52.712 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:52 smithi078 ceph-mon[23689]: from='client.14732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:39:52.713 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:52 smithi078 ceph-mon[23689]: from='client.14736 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:39:52.713 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:52 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/1717234311' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:39:52.713 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:52 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:52.713 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:52 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:52.760 INFO:teuthology.orchestra.run.smithi078.stdout: 5 96529 13.15 MB/sec execute 47 sec latency 48.502 ms 2024-09-17T09:39:52.904 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:39:52.904 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407", 2024-09-17T09:39:52.904 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-17T09:39:52.904 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T09:39:52.904 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-09-17T09:39:52.904 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "1/23 daemons upgraded", 2024-09-17T09:39:52.905 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "", 2024-09-17T09:39:52.905 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-17T09:39:52.905 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:39:52.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:52 smithi136 ceph-mon[25953]: from='client.14732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:39:52.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:52 smithi136 ceph-mon[25953]: from='client.14736 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:39:52.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:52 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/1717234311' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:39:52.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:52 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:52.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:52 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:39:53.599 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-17T09:39:53.717 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:53 smithi078 ceph-mon[23689]: pgmap v30: 97 pgs: 97 active+clean; 326 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.7 MiB/s wr, 574 op/s 2024-09-17T09:39:53.717 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:53 smithi078 ceph-mon[23689]: Reconfiguring prometheus.smithi078 (dependencies changed)... 2024-09-17T09:39:53.718 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:53 smithi078 ceph-mon[23689]: Deploying daemon prometheus.smithi078 on smithi078 2024-09-17T09:39:53.768 INFO:teuthology.orchestra.run.smithi078.stdout: 5 96985 13.05 MB/sec execute 48 sec latency 31.932 ms 2024-09-17T09:39:53.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:53 smithi136 ceph-mon[25953]: pgmap v30: 97 pgs: 97 active+clean; 326 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.7 MiB/s wr, 574 op/s 2024-09-17T09:39:53.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:53 smithi136 ceph-mon[25953]: Reconfiguring prometheus.smithi078 (dependencies changed)... 2024-09-17T09:39:53.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:53 smithi136 ceph-mon[25953]: Deploying daemon prometheus.smithi078 on smithi078 2024-09-17T09:39:54.724 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:54 smithi078 ceph-mon[23689]: from='client.14744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:39:54.724 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:54 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/1702068771' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:39:54.775 INFO:teuthology.orchestra.run.smithi078.stdout: 5 97441 12.94 MB/sec execute 49 sec latency 50.631 ms 2024-09-17T09:39:54.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:54 smithi136 ceph-mon[25953]: from='client.14744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:39:54.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:54 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/1702068771' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:39:55.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:55 smithi078 ceph-mon[23689]: pgmap v31: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 7.5 MiB/s wr, 798 op/s 2024-09-17T09:39:55.776 INFO:teuthology.orchestra.run.smithi078.stdout: 5 97972 12.96 MB/sec execute 50 sec latency 58.863 ms 2024-09-17T09:39:55.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:55 smithi136 ceph-mon[25953]: pgmap v31: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 7.5 MiB/s wr, 798 op/s 2024-09-17T09:39:56.776 INFO:teuthology.orchestra.run.smithi078.stdout: 5 98564 13.08 MB/sec execute 51 sec latency 29.900 ms 2024-09-17T09:39:57.776 INFO:teuthology.orchestra.run.smithi078.stdout: 5 98941 13.05 MB/sec execute 52 sec latency 45.266 ms 2024-09-17T09:39:57.776 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:57 smithi078 ceph-mon[23689]: pgmap v32: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 721 op/s 2024-09-17T09:39:57.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:57 smithi136 ceph-mon[25953]: pgmap v32: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 721 op/s 2024-09-17T09:39:58.776 INFO:teuthology.orchestra.run.smithi078.stdout: 5 99223 13.01 MB/sec execute 53 sec latency 56.360 ms 2024-09-17T09:39:59.782 INFO:teuthology.orchestra.run.smithi078.stdout: 5 99645 13.01 MB/sec execute 54 sec latency 73.132 ms 2024-09-17T09:39:59.798 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:59 smithi078 ceph-mon[23689]: pgmap v33: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.1 MiB/s wr, 596 op/s 2024-09-17T09:39:59.798 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:39:59 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T09:39:59.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:59 smithi136 ceph-mon[25953]: pgmap v33: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.1 MiB/s wr, 596 op/s 2024-09-17T09:39:59.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:39:59 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T09:40:00.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 100061 12.94 MB/sec execute 55 sec latency 35.402 ms 2024-09-17T09:40:00.804 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:00 smithi078 ceph-mon[23689]: overall HEALTH_OK 2024-09-17T09:40:00.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:00 smithi136 ceph-mon[25953]: overall HEALTH_OK 2024-09-17T09:40:01.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:01 smithi078 ceph-mon[23689]: pgmap v34: 97 pgs: 97 active+clean; 326 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 852 op/s 2024-09-17T09:40:01.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 100371 12.82 MB/sec execute 56 sec latency 49.955 ms 2024-09-17T09:40:01.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:01 smithi136 ceph-mon[25953]: pgmap v34: 97 pgs: 97 active+clean; 326 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 852 op/s 2024-09-17T09:40:02.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 100710 12.70 MB/sec execute 57 sec latency 72.955 ms 2024-09-17T09:40:03.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 101002 12.58 MB/sec execute 58 sec latency 72.535 ms 2024-09-17T09:40:03.845 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:03 smithi078 ceph-mon[23689]: pgmap v35: 97 pgs: 97 active+clean; 326 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.1 MiB/s wr, 571 op/s 2024-09-17T09:40:03.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:03 smithi136 ceph-mon[25953]: pgmap v35: 97 pgs: 97 active+clean; 326 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.1 MiB/s wr, 571 op/s 2024-09-17T09:40:04.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 101343 12.54 MB/sec execute 59 sec latency 53.941 ms 2024-09-17T09:40:05.673 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:05 smithi078 ceph-mon[23689]: pgmap v36: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.4 MiB/s wr, 594 op/s 2024-09-17T09:40:05.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 101587 12.40 MB/sec execute 60 sec latency 58.208 ms 2024-09-17T09:40:05.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:05 smithi136 ceph-mon[25953]: pgmap v36: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.4 MiB/s wr, 594 op/s 2024-09-17T09:40:06.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 101925 12.38 MB/sec execute 61 sec latency 59.481 ms 2024-09-17T09:40:07.751 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:07 smithi078 ceph-mon[23689]: pgmap v37: 97 pgs: 97 active+clean; 323 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 6.4 MiB/s wr, 574 op/s 2024-09-17T09:40:07.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 102261 12.36 MB/sec execute 62 sec latency 72.710 ms 2024-09-17T09:40:07.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:07 smithi136 ceph-mon[25953]: pgmap v37: 97 pgs: 97 active+clean; 323 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 6.4 MiB/s wr, 574 op/s 2024-09-17T09:40:08.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 102810 12.51 MB/sec execute 63 sec latency 19.990 ms 2024-09-17T09:40:09.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 103301 12.50 MB/sec execute 64 sec latency 38.497 ms 2024-09-17T09:40:09.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:09 smithi078 ceph-mon[23689]: pgmap v38: 97 pgs: 97 active+clean; 323 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.8 MiB/s wr, 483 op/s 2024-09-17T09:40:09.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:09 smithi136 ceph-mon[25953]: pgmap v38: 97 pgs: 97 active+clean; 323 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.8 MiB/s wr, 483 op/s 2024-09-17T09:40:10.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 103750 12.49 MB/sec execute 65 sec latency 28.792 ms 2024-09-17T09:40:11.785 INFO:teuthology.orchestra.run.smithi078.stdout: 5 104287 12.45 MB/sec execute 66 sec latency 34.132 ms 2024-09-17T09:40:11.823 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:11 smithi078 ceph-mon[23689]: pgmap v39: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.4 MiB/s wr, 697 op/s 2024-09-17T09:40:11.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:11 smithi136 ceph-mon[25953]: pgmap v39: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.4 MiB/s wr, 697 op/s 2024-09-17T09:40:12.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 104748 12.40 MB/sec execute 67 sec latency 64.568 ms 2024-09-17T09:40:13.785 INFO:teuthology.orchestra.run.smithi078.stdout: 5 105266 12.41 MB/sec execute 68 sec latency 39.348 ms 2024-09-17T09:40:13.870 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:13 smithi078 ceph-mon[23689]: pgmap v40: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.6 MiB/s wr, 441 op/s 2024-09-17T09:40:13.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:13 smithi136 ceph-mon[25953]: pgmap v40: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.6 MiB/s wr, 441 op/s 2024-09-17T09:40:14.785 INFO:teuthology.orchestra.run.smithi078.stdout: 5 105748 12.47 MB/sec execute 69 sec latency 60.161 ms 2024-09-17T09:40:14.888 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:14 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T09:40:14.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:14 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T09:40:15.654 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:15 smithi078 ceph-mon[23689]: pgmap v41: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.4 MiB/s wr, 494 op/s 2024-09-17T09:40:15.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 106170 12.50 MB/sec execute 70 sec latency 32.518 ms 2024-09-17T09:40:15.913 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:15 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:15.913 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:15 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:15.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:15 smithi136 ceph-mon[25953]: pgmap v41: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.4 MiB/s wr, 494 op/s 2024-09-17T09:40:15.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:15 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:15.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:15 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:16.675 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:16 smithi078 ceph-mon[23689]: Reconfiguring node-exporter.smithi136 (dependencies changed)... 2024-09-17T09:40:16.675 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:16 smithi078 ceph-mon[23689]: Deploying daemon node-exporter.smithi136 on smithi136 2024-09-17T09:40:16.785 INFO:teuthology.orchestra.run.smithi078.stdout: 5 106551 12.52 MB/sec execute 71 sec latency 38.233 ms 2024-09-17T09:40:16.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:16 smithi136 ceph-mon[25953]: Reconfiguring node-exporter.smithi136 (dependencies changed)... 2024-09-17T09:40:16.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:16 smithi136 ceph-mon[25953]: Deploying daemon node-exporter.smithi136 on smithi136 2024-09-17T09:40:17.696 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:17 smithi078 ceph-mon[23689]: pgmap v42: 97 pgs: 97 active+clean; 322 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 713 op/s 2024-09-17T09:40:17.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 106860 12.45 MB/sec execute 72 sec latency 78.479 ms 2024-09-17T09:40:17.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:17 smithi136 ceph-mon[25953]: pgmap v42: 97 pgs: 97 active+clean; 322 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 713 op/s 2024-09-17T09:40:18.785 INFO:teuthology.orchestra.run.smithi078.stdout: 5 107306 12.43 MB/sec execute 73 sec latency 84.632 ms 2024-09-17T09:40:19.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:19 smithi078 ceph-mon[23689]: pgmap v43: 97 pgs: 97 active+clean; 322 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.2 MiB/s wr, 509 op/s 2024-09-17T09:40:19.785 INFO:teuthology.orchestra.run.smithi078.stdout: 5 107841 12.41 MB/sec execute 74 sec latency 35.477 ms 2024-09-17T09:40:19.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:19 smithi136 ceph-mon[25953]: pgmap v43: 97 pgs: 97 active+clean; 322 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.2 MiB/s wr, 509 op/s 2024-09-17T09:40:20.785 INFO:teuthology.orchestra.run.smithi078.stdout: 5 108312 12.34 MB/sec execute 75 sec latency 78.777 ms 2024-09-17T09:40:21.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:21 smithi136 ceph-mon[25953]: pgmap v44: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.0 MiB/s wr, 760 op/s 2024-09-17T09:40:21.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:21 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:21.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:21 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:21.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:21 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T09:40:21.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:21 smithi136 ceph-mon[25953]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T09:40:21.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:21 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi078.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T09:40:21.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:21 smithi136 ceph-mon[25953]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi078.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T09:40:21.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:21 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:21.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:21 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T09:40:21.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:21 smithi136 ceph-mon[25953]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T09:40:21.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:21 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi078.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T09:40:21.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:21 smithi136 ceph-mon[25953]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi078.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T09:40:21.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:21 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:21.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:21 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:40:21.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:21 smithi136 ceph-mon[25953]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:40:21.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:21 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi078.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T09:40:21.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:21 smithi136 ceph-mon[25953]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi078.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T09:40:21.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:21 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:21.728 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:21 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T09:40:21.728 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:21 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:40:21.728 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:21 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T09:40:21.779 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:21 smithi078 ceph-mon[23689]: pgmap v44: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 8.0 MiB/s wr, 760 op/s 2024-09-17T09:40:21.779 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:21 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:21.779 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:21 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:21.779 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:21 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T09:40:21.780 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:21 smithi078 ceph-mon[23689]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T09:40:21.780 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:21 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi078.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T09:40:21.780 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:21 smithi078 ceph-mon[23689]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi078.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T09:40:21.780 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:21 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:21.780 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:21 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T09:40:21.780 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:21 smithi078 ceph-mon[23689]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T09:40:21.780 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:21 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi078.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T09:40:21.781 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:21 smithi078 ceph-mon[23689]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi078.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T09:40:21.781 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:21 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:21.781 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:21 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:40:21.781 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:21 smithi078 ceph-mon[23689]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:40:21.781 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:21 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi078.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T09:40:21.781 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:21 smithi078 ceph-mon[23689]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi078.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T09:40:21.781 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:21 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:21.781 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:21 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T09:40:21.781 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:21 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:40:21.781 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:21 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T09:40:21.785 INFO:teuthology.orchestra.run.smithi078.stdout: 5 108716 12.34 MB/sec execute 76 sec latency 65.532 ms 2024-09-17T09:40:22.785 INFO:teuthology.orchestra.run.smithi078.stdout: 5 109257 12.42 MB/sec execute 77 sec latency 38.426 ms 2024-09-17T09:40:22.785 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:22 smithi078 ceph-mon[23689]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T09:40:22.785 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:22 smithi078 ceph-mon[23689]: Upgrade: Need to upgrade myself (mgr.smithi136.hicmlm) 2024-09-17T09:40:22.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:22 smithi078 ceph-mon[23689]: Upgrade: Need to upgrade myself (mgr.smithi136.hicmlm) 2024-09-17T09:40:22.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:22 smithi078 ceph-mon[23689]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T09:40:22.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:22 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T09:40:22.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:22 smithi136 ceph-mon[25953]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T09:40:22.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:22 smithi136 ceph-mon[25953]: Upgrade: Need to upgrade myself (mgr.smithi136.hicmlm) 2024-09-17T09:40:22.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:22 smithi136 ceph-mon[25953]: Upgrade: Need to upgrade myself (mgr.smithi136.hicmlm) 2024-09-17T09:40:22.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:22 smithi136 ceph-mon[25953]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T09:40:22.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:22 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T09:40:23.785 INFO:teuthology.orchestra.run.smithi078.stdout: 5 109774 12.49 MB/sec execute 78 sec latency 33.676 ms 2024-09-17T09:40:23.791 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:23 smithi078 ceph-mon[23689]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T09:40:23.791 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:23 smithi078 ceph-mon[23689]: pgmap v45: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.5 MiB/s wr, 546 op/s 2024-09-17T09:40:23.791 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:23 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:23.791 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:23 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.rsqofi", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T09:40:23.792 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:23 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.rsqofi", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T09:40:23.792 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:23 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T09:40:23.792 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:23 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:40:23.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:23 smithi136 ceph-mon[25953]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T09:40:23.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:23 smithi136 ceph-mon[25953]: pgmap v45: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.5 MiB/s wr, 546 op/s 2024-09-17T09:40:23.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:23 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:23.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:23 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.rsqofi", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T09:40:23.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:23 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.rsqofi", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T09:40:23.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:23 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T09:40:23.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:23 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:40:24.206 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-17T09:40:24.785 INFO:teuthology.orchestra.run.smithi078.stdout: 5 110193 12.50 MB/sec execute 79 sec latency 36.145 ms 2024-09-17T09:40:24.805 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:24 smithi078 ceph-mon[23689]: Upgrade: Updating mgr.smithi078.rsqofi 2024-09-17T09:40:24.805 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:24 smithi078 ceph-mon[23689]: Deploying daemon mgr.smithi078.rsqofi on smithi078 2024-09-17T09:40:24.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:24 smithi136 ceph-mon[25953]: Upgrade: Updating mgr.smithi078.rsqofi 2024-09-17T09:40:24.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:24 smithi136 ceph-mon[25953]: Deploying daemon mgr.smithi078.rsqofi on smithi078 2024-09-17T09:40:25.193 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:40:25.193 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 starting - - - - 2024-09-17T09:40:25.193 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (9m) 74s ago 9m 6400k - 17.2.0 e1d6a67b021e 46d44333624b 2024-09-17T09:40:25.193 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi136 smithi136 running (9m) 74s ago 9m 6395k - 17.2.0 e1d6a67b021e 56ce035cb482 2024-09-17T09:40:25.193 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 starting - - - - 2024-09-17T09:40:25.193 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.jdrsup smithi078 running (3m) 74s ago 3m 37.0M - 17.2.0 e1d6a67b021e 8f834e15ba88 2024-09-17T09:40:25.193 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi136.smslff smithi136 running (3m) 74s ago 3m 14.4M - 17.2.0 e1d6a67b021e 0c20e2b328d3 2024-09-17T09:40:25.193 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rsqofi smithi078 *:9283 running (11m) 74s ago 11m 166M - 17.2.0 e1d6a67b021e f2012c57d1ac 2024-09-17T09:40:25.194 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi136.hicmlm smithi136 *:8443,9283 running (97s) 74s ago 9m 497M - 19.3.0-4963-ge646f729 f271290a0950 bbd092a7e4ca 2024-09-17T09:40:25.194 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (11m) 74s ago 11m 54.8M 2048M 17.2.0 e1d6a67b021e 09299430e248 2024-09-17T09:40:25.194 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi136 smithi136 running (9m) 74s ago 9m 38.3M 2048M 17.2.0 e1d6a67b021e 1bfdfb0617bb 2024-09-17T09:40:25.194 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.tcwbvi smithi078 *:2049 running (3m) 74s ago 3m 262M - 4.0 e1d6a67b021e 30c87cb46555 2024-09-17T09:40:25.194 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi136.regjvc smithi136 *:2049 running (3m) 74s ago 3m 77.5M - 4.0 e1d6a67b021e b88e4dc24376 2024-09-17T09:40:25.194 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 starting - - - - 2024-09-17T09:40:25.194 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi136 smithi136 *:9100 starting - - - - 2024-09-17T09:40:25.194 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (8m) 74s ago 8m 292M 1936M 17.2.0 e1d6a67b021e b4ca42f8477d 2024-09-17T09:40:25.194 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (7m) 74s ago 7m 236M 1936M 17.2.0 e1d6a67b021e 351873014de4 2024-09-17T09:40:25.194 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (7m) 74s ago 7m 176M 1936M 17.2.0 e1d6a67b021e 07eb71e4fa57 2024-09-17T09:40:25.194 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (6m) 74s ago 6m 250M 1936M 17.2.0 e1d6a67b021e 1b3a8be89881 2024-09-17T09:40:25.194 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi136 running (6m) 74s ago 6m 263M 2704M 17.2.0 e1d6a67b021e a0c5b4de2414 2024-09-17T09:40:25.194 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi136 running (6m) 74s ago 6m 297M 2704M 17.2.0 e1d6a67b021e 30a3049955f9 2024-09-17T09:40:25.195 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi136 running (5m) 74s ago 5m 267M 2704M 17.2.0 e1d6a67b021e 1698e1567c33 2024-09-17T09:40:25.195 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi136 running (5m) 74s ago 5m 373M 2704M 17.2.0 e1d6a67b021e 5d3b08cecd78 2024-09-17T09:40:25.195 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 starting - - - - 2024-09-17T09:40:25.664 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:25 smithi078 ceph-mon[23689]: pgmap v46: 97 pgs: 97 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 7.1 MiB/s wr, 707 op/s 2024-09-17T09:40:25.664 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:25 smithi078 ceph-mon[23689]: from='client.14752 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:40:25.743 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:40:25.743 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-17T09:40:25.743 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:40:25.743 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:40:25.743 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-17T09:40:25.743 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T09:40:25.743 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 1 2024-09-17T09:40:25.743 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:40:25.743 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-17T09:40:25.744 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T09:40:25.744 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:40:25.744 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-17T09:40:25.744 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:40:25.744 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:40:25.744 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-17T09:40:25.744 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-17T09:40:25.744 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 1 2024-09-17T09:40:25.744 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-17T09:40:25.744 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:40:25.785 INFO:teuthology.orchestra.run.smithi078.stdout: 5 110499 12.43 MB/sec execute 80 sec latency 88.026 ms 2024-09-17T09:40:25.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:25 smithi136 ceph-mon[25953]: pgmap v46: 97 pgs: 97 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 7.1 MiB/s wr, 707 op/s 2024-09-17T09:40:25.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:25 smithi136 ceph-mon[25953]: from='client.14752 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:40:26.359 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:40:26.360 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407", 2024-09-17T09:40:26.360 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-17T09:40:26.360 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T09:40:26.360 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-09-17T09:40:26.360 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "1/23 daemons upgraded", 2024-09-17T09:40:26.360 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading mgr daemons", 2024-09-17T09:40:26.360 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-17T09:40:26.360 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:40:26.785 INFO:teuthology.orchestra.run.smithi078.stdout: 5 110796 12.34 MB/sec execute 81 sec latency 38.545 ms 2024-09-17T09:40:26.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:26 smithi078 ceph-mon[23689]: from='client.14756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:40:26.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:26 smithi078 ceph-mon[23689]: from='client.24495 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:40:26.856 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:26 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/2689339518' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:40:26.931 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-17T09:40:26.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:26 smithi136 ceph-mon[25953]: from='client.14756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:40:26.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:26 smithi136 ceph-mon[25953]: from='client.24495 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:40:26.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:26 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/2689339518' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:40:27.785 INFO:teuthology.orchestra.run.smithi078.stdout: 5 111092 12.27 MB/sec execute 82 sec latency 56.013 ms 2024-09-17T09:40:27.865 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:27 smithi078 ceph-mon[23689]: pgmap v47: 97 pgs: 97 active+clean; 328 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 783 op/s 2024-09-17T09:40:27.866 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:27 smithi078 ceph-mon[23689]: from='client.14768 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:40:27.866 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:27 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/873031266' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:40:27.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:27 smithi136 ceph-mon[25953]: pgmap v47: 97 pgs: 97 active+clean; 328 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 783 op/s 2024-09-17T09:40:27.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:27 smithi136 ceph-mon[25953]: from='client.14768 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:40:27.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:27 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/873031266' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:40:28.785 INFO:teuthology.orchestra.run.smithi078.stdout: 5 111527 12.24 MB/sec execute 83 sec latency 33.615 ms 2024-09-17T09:40:29.782 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:29 smithi078 ceph-mon[23689]: pgmap v48: 97 pgs: 97 active+clean; 328 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.9 MiB/s wr, 541 op/s 2024-09-17T09:40:29.783 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:29 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T09:40:29.785 INFO:teuthology.orchestra.run.smithi078.stdout: 5 111987 12.19 MB/sec execute 84 sec latency 45.671 ms 2024-09-17T09:40:29.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:29 smithi136 ceph-mon[25953]: pgmap v48: 97 pgs: 97 active+clean; 328 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.9 MiB/s wr, 541 op/s 2024-09-17T09:40:29.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:29 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T09:40:30.785 INFO:teuthology.orchestra.run.smithi078.stdout: 5 112416 12.20 MB/sec execute 85 sec latency 33.550 ms 2024-09-17T09:40:31.565 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:31 smithi078 ceph-mon[23689]: pgmap v49: 97 pgs: 97 active+clean; 320 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 764 op/s 2024-09-17T09:40:31.566 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:31 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:31.566 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:31 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:31.566 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:31 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:40:31.566 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:31 smithi078 ceph-mon[23689]: mgrmap e28: smithi136.hicmlm(active, since 93s), standbys: smithi078.rsqofi 2024-09-17T09:40:31.566 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:31 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:31.566 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:31 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:31.785 INFO:teuthology.orchestra.run.smithi078.stdout: 5 112829 12.23 MB/sec execute 86 sec latency 53.241 ms 2024-09-17T09:40:31.859 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:31 smithi136 ceph-mon[25953]: pgmap v49: 97 pgs: 97 active+clean; 320 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 764 op/s 2024-09-17T09:40:31.859 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:31 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:31.859 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:31 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:31.859 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:31 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:40:31.860 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:31 smithi136 ceph-mon[25953]: mgrmap e28: smithi136.hicmlm(active, since 93s), standbys: smithi078.rsqofi 2024-09-17T09:40:31.860 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:31 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:31.860 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:31 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:32.785 INFO:teuthology.orchestra.run.smithi078.stdout: 5 113254 12.27 MB/sec execute 87 sec latency 26.348 ms 2024-09-17T09:40:33.595 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:33 smithi078 ceph-mon[23689]: pgmap v50: 97 pgs: 97 active+clean; 320 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.0 MiB/s wr, 513 op/s 2024-09-17T09:40:33.785 INFO:teuthology.orchestra.run.smithi078.stdout: 5 113624 12.29 MB/sec execute 88 sec latency 43.770 ms 2024-09-17T09:40:33.795 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:33 smithi136 ceph-mon[25953]: pgmap v50: 97 pgs: 97 active+clean; 320 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.0 MiB/s wr, 513 op/s 2024-09-17T09:40:34.785 INFO:teuthology.orchestra.run.smithi078.stdout: 5 113869 12.21 MB/sec execute 89 sec latency 126.701 ms 2024-09-17T09:40:35.243 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:35 smithi078 ceph-mon[23689]: pgmap v51: 97 pgs: 97 active+clean; 329 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.6 MiB/s wr, 636 op/s 2024-09-17T09:40:35.243 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:35 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:35.243 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:35 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:35.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:35 smithi136 ceph-mon[25953]: pgmap v51: 97 pgs: 97 active+clean; 329 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.6 MiB/s wr, 636 op/s 2024-09-17T09:40:35.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:35 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:35.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:35 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:35.785 INFO:teuthology.orchestra.run.smithi078.stdout: 5 114178 12.14 MB/sec execute 90 sec latency 189.285 ms 2024-09-17T09:40:36.785 INFO:teuthology.orchestra.run.smithi078.stdout: 5 114478 12.07 MB/sec execute 91 sec latency 42.270 ms 2024-09-17T09:40:37.170 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:37 smithi078 ceph-mon[23689]: pgmap v52: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.0 MiB/s wr, 584 op/s 2024-09-17T09:40:37.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:37 smithi136 ceph-mon[25953]: pgmap v52: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.0 MiB/s wr, 584 op/s 2024-09-17T09:40:37.785 INFO:teuthology.orchestra.run.smithi078.stdout: 5 114945 12.07 MB/sec execute 92 sec latency 39.683 ms 2024-09-17T09:40:38.185 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:38 smithi078 ceph-mon[23689]: Standby manager daemon smithi078.rsqofi restarted 2024-09-17T09:40:38.185 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:38 smithi078 ceph-mon[23689]: Standby manager daemon smithi078.rsqofi started 2024-09-17T09:40:38.185 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:38 smithi078 ceph-mon[23689]: from='mgr.? 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.rsqofi/crt"}]: dispatch 2024-09-17T09:40:38.186 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:38 smithi078 ceph-mon[23689]: from='mgr.? 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T09:40:38.186 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:38 smithi078 ceph-mon[23689]: from='mgr.? 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.rsqofi/key"}]: dispatch 2024-09-17T09:40:38.186 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:38 smithi078 ceph-mon[23689]: from='mgr.? 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T09:40:38.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:38 smithi136 ceph-mon[25953]: Standby manager daemon smithi078.rsqofi restarted 2024-09-17T09:40:38.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:38 smithi136 ceph-mon[25953]: Standby manager daemon smithi078.rsqofi started 2024-09-17T09:40:38.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:38 smithi136 ceph-mon[25953]: from='mgr.? 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.rsqofi/crt"}]: dispatch 2024-09-17T09:40:38.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:38 smithi136 ceph-mon[25953]: from='mgr.? 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T09:40:38.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:38 smithi136 ceph-mon[25953]: from='mgr.? 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.rsqofi/key"}]: dispatch 2024-09-17T09:40:38.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:38 smithi136 ceph-mon[25953]: from='mgr.? 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T09:40:38.786 INFO:teuthology.orchestra.run.smithi078.stdout: 5 115498 12.06 MB/sec execute 93 sec latency 34.941 ms 2024-09-17T09:40:39.196 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:39 smithi078 ceph-mon[23689]: pgmap v53: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.7 MiB/s wr, 454 op/s 2024-09-17T09:40:39.196 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:39 smithi078 ceph-mon[23689]: mgrmap e29: smithi136.hicmlm(active, since 101s), standbys: smithi078.rsqofi 2024-09-17T09:40:39.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:39 smithi136 ceph-mon[25953]: pgmap v53: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.7 MiB/s wr, 454 op/s 2024-09-17T09:40:39.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:39 smithi136 ceph-mon[25953]: mgrmap e29: smithi136.hicmlm(active, since 101s), standbys: smithi078.rsqofi 2024-09-17T09:40:39.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 116181 12.13 MB/sec execute 94 sec latency 22.533 ms 2024-09-17T09:40:40.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 116959 12.29 MB/sec execute 95 sec latency 84.929 ms 2024-09-17T09:40:41.205 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:41 smithi078 ceph-mon[23689]: pgmap v54: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 701 op/s 2024-09-17T09:40:41.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:41 smithi136 ceph-mon[25953]: pgmap v54: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 701 op/s 2024-09-17T09:40:41.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 117540 12.32 MB/sec execute 96 sec latency 17.624 ms 2024-09-17T09:40:42.715 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:42 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:42.715 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:42 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:42.715 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:42 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:42.716 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:42 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:42.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 117970 12.31 MB/sec execute 97 sec latency 42.931 ms 2024-09-17T09:40:42.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:42 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:42.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:42 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:42.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:42 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:42.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:42 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:43.722 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:43 smithi078 ceph-mon[23689]: pgmap v55: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.9 MiB/s wr, 478 op/s 2024-09-17T09:40:43.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 118470 12.30 MB/sec execute 98 sec latency 35.785 ms 2024-09-17T09:40:43.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:43 smithi136 ceph-mon[25953]: pgmap v55: 97 pgs: 97 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.9 MiB/s wr, 478 op/s 2024-09-17T09:40:44.736 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:44 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T09:40:44.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 119008 12.25 MB/sec execute 99 sec latency 73.066 ms 2024-09-17T09:40:44.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:44 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T09:40:45.746 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:45 smithi078 ceph-mon[23689]: pgmap v56: 97 pgs: 97 active+clean; 323 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 7.1 MiB/s wr, 627 op/s 2024-09-17T09:40:45.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 119550 12.29 MB/sec execute 100 sec latency 37.415 ms 2024-09-17T09:40:45.876 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:45 smithi136 ceph-mon[25953]: pgmap v56: 97 pgs: 97 active+clean; 323 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 7.1 MiB/s wr, 627 op/s 2024-09-17T09:40:46.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 120526 12.52 MB/sec execute 101 sec latency 21.757 ms 2024-09-17T09:40:47.755 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:47 smithi078 ceph-mon[23689]: pgmap v57: 97 pgs: 97 active+clean; 328 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 703 op/s 2024-09-17T09:40:47.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 121105 12.53 MB/sec execute 102 sec latency 39.373 ms 2024-09-17T09:40:47.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:47 smithi136 ceph-mon[25953]: pgmap v57: 97 pgs: 97 active+clean; 328 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 703 op/s 2024-09-17T09:40:48.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 121638 12.54 MB/sec execute 103 sec latency 29.793 ms 2024-09-17T09:40:49.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 122098 12.52 MB/sec execute 104 sec latency 40.938 ms 2024-09-17T09:40:49.794 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:49 smithi078 ceph-mon[23689]: pgmap v58: 97 pgs: 97 active+clean; 328 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.9 MiB/s wr, 594 op/s 2024-09-17T09:40:49.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:49 smithi136 ceph-mon[25953]: pgmap v58: 97 pgs: 97 active+clean; 328 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.9 MiB/s wr, 594 op/s 2024-09-17T09:40:50.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 122542 12.47 MB/sec execute 105 sec latency 48.444 ms 2024-09-17T09:40:50.825 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:50 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:50.825 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:50 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:50.825 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:50 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:40:50.825 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:50 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:40:50.825 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:50 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:50.825 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:50 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:50.825 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:50 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:40:50.825 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:50 smithi078 ceph-mon[23689]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "mgr fail", "who": "smithi136.hicmlm"}]: dispatch 2024-09-17T09:40:50.825 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:50 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "mgr fail", "who": "smithi136.hicmlm"}]: dispatch 2024-09-17T09:40:50.825 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:50 smithi078 ceph-mon[23689]: osdmap e72: 8 total, 8 up, 8 in 2024-09-17T09:40:50.826 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:50 smithi078 ceph-mon[23689]: Standby manager daemon smithi136.hicmlm started 2024-09-17T09:40:50.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:50 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:50.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:50 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:50.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:50 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:40:50.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:50 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:40:50.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:50 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:50.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:50 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' 2024-09-17T09:40:50.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:50 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:40:50.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:50 smithi136 ceph-mon[25953]: from='mgr.24419 172.21.15.136:0/2809782326' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "mgr fail", "who": "smithi136.hicmlm"}]: dispatch 2024-09-17T09:40:50.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:50 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "mgr fail", "who": "smithi136.hicmlm"}]: dispatch 2024-09-17T09:40:50.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:50 smithi136 ceph-mon[25953]: osdmap e72: 8 total, 8 up, 8 in 2024-09-17T09:40:50.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:50 smithi136 ceph-mon[25953]: Standby manager daemon smithi136.hicmlm started 2024-09-17T09:40:51.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 122849 12.43 MB/sec execute 106 sec latency 31.046 ms 2024-09-17T09:40:51.809 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:51 smithi078 ceph-mon[23689]: Upgrade: Need to upgrade myself (mgr.smithi136.hicmlm) 2024-09-17T09:40:51.809 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:51 smithi078 ceph-mon[23689]: pgmap v59: 97 pgs: 97 active+clean; 332 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 9.9 MiB/s wr, 963 op/s 2024-09-17T09:40:51.809 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:51 smithi078 ceph-mon[23689]: Upgrade: Need to upgrade myself (mgr.smithi136.hicmlm) 2024-09-17T09:40:51.809 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:51 smithi078 ceph-mon[23689]: Failing over to other MGR 2024-09-17T09:40:51.810 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:51 smithi078 ceph-mon[23689]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' cmd='[{"prefix": "mgr fail", "who": "smithi136.hicmlm"}]': finished 2024-09-17T09:40:51.810 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:51 smithi078 ceph-mon[23689]: mgrmap e30: smithi078.rsqofi(active, starting, since 0.609411s), standbys: smithi136.hicmlm 2024-09-17T09:40:51.810 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:51 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-17T09:40:51.810 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:51 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata", "id": "smithi136"}]: dispatch 2024-09-17T09:40:51.810 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:51 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.jdrsup"}]: dispatch 2024-09-17T09:40:51.810 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:51 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi136.smslff"}]: dispatch 2024-09-17T09:40:51.810 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:51 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mgr metadata", "who": "smithi078.rsqofi", "id": "smithi078.rsqofi"}]: dispatch 2024-09-17T09:40:51.810 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:51 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mgr metadata", "who": "smithi136.hicmlm", "id": "smithi136.hicmlm"}]: dispatch 2024-09-17T09:40:51.810 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:51 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:40:51.810 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:51 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:40:51.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:51 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T09:40:51.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:51 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:40:51.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:51 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:40:51.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:51 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T09:40:51.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:51 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T09:40:51.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:51 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T09:40:51.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:51 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T09:40:51.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:51 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T09:40:51.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:51 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T09:40:51.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:51 smithi078 ceph-mon[23689]: Manager daemon smithi078.rsqofi is now available 2024-09-17T09:40:51.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:51 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:40:51.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:51 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T09:40:51.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:51 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T09:40:51.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:51 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:40:51.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:51 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/trash_purge_schedule"}]: dispatch 2024-09-17T09:40:51.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:51 smithi136 ceph-mon[25953]: Upgrade: Need to upgrade myself (mgr.smithi136.hicmlm) 2024-09-17T09:40:51.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:51 smithi136 ceph-mon[25953]: pgmap v59: 97 pgs: 97 active+clean; 332 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 9.9 MiB/s wr, 963 op/s 2024-09-17T09:40:51.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:51 smithi136 ceph-mon[25953]: Upgrade: Need to upgrade myself (mgr.smithi136.hicmlm) 2024-09-17T09:40:51.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:51 smithi136 ceph-mon[25953]: Failing over to other MGR 2024-09-17T09:40:51.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:51 smithi136 ceph-mon[25953]: from='mgr.24419 ' entity='mgr.smithi136.hicmlm' cmd='[{"prefix": "mgr fail", "who": "smithi136.hicmlm"}]': finished 2024-09-17T09:40:51.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:51 smithi136 ceph-mon[25953]: mgrmap e30: smithi078.rsqofi(active, starting, since 0.609411s), standbys: smithi136.hicmlm 2024-09-17T09:40:51.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:51 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-17T09:40:51.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:51 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata", "id": "smithi136"}]: dispatch 2024-09-17T09:40:51.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:51 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.jdrsup"}]: dispatch 2024-09-17T09:40:51.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:51 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi136.smslff"}]: dispatch 2024-09-17T09:40:51.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:51 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mgr metadata", "who": "smithi078.rsqofi", "id": "smithi078.rsqofi"}]: dispatch 2024-09-17T09:40:51.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:51 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mgr metadata", "who": "smithi136.hicmlm", "id": "smithi136.hicmlm"}]: dispatch 2024-09-17T09:40:51.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:51 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T09:40:51.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:51 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T09:40:51.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:51 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T09:40:51.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:51 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T09:40:51.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:51 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T09:40:51.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:51 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T09:40:51.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:51 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T09:40:51.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:51 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T09:40:51.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:51 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T09:40:51.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:51 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T09:40:51.978 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:51 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T09:40:51.978 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:51 smithi136 ceph-mon[25953]: Manager daemon smithi078.rsqofi is now available 2024-09-17T09:40:51.978 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:51 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:40:51.978 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:51 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T09:40:51.978 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:51 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T09:40:51.978 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:51 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/mirror_snapshot_schedule"}]: dispatch 2024-09-17T09:40:51.978 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:51 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/trash_purge_schedule"}]: dispatch 2024-09-17T09:40:52.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 123467 12.52 MB/sec execute 107 sec latency 48.681 ms 2024-09-17T09:40:52.825 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:52 smithi078 ceph-mon[23689]: mgrmap e31: smithi078.rsqofi(active, since 1.61958s), standbys: smithi136.hicmlm 2024-09-17T09:40:52.826 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:52 smithi078 ceph-mon[23689]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T09:40:52.826 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:52 smithi078 ceph-mon[23689]: pgmap v3: 97 pgs: 97 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-17T09:40:53.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:52 smithi136 ceph-mon[25953]: mgrmap e31: smithi078.rsqofi(active, since 1.61958s), standbys: smithi136.hicmlm 2024-09-17T09:40:53.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:52 smithi136 ceph-mon[25953]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T09:40:53.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:52 smithi136 ceph-mon[25953]: pgmap v3: 97 pgs: 97 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-17T09:40:53.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 123892 12.55 MB/sec execute 108 sec latency 47.315 ms 2024-09-17T09:40:53.824 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:53 smithi136 ceph-mon[25953]: pgmap v4: 97 pgs: 97 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-17T09:40:53.824 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:53 smithi136 ceph-mon[25953]: mgrmap e32: smithi078.rsqofi(active, since 2s), standbys: smithi136.hicmlm 2024-09-17T09:40:53.840 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:53 smithi078 ceph-mon[23689]: pgmap v4: 97 pgs: 97 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-17T09:40:53.840 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:53 smithi078 ceph-mon[23689]: mgrmap e32: smithi078.rsqofi(active, since 2s), standbys: smithi136.hicmlm 2024-09-17T09:40:54.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 124108 12.49 MB/sec execute 109 sec latency 51.418 ms 2024-09-17T09:40:54.867 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:54 smithi078 ceph-mon[23689]: [17/Sep/2024:09:40:53] ENGINE Bus STARTING 2024-09-17T09:40:54.867 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:54 smithi078 ceph-mon[23689]: [17/Sep/2024:09:40:53] ENGINE Serving on http://172.21.15.78:8765 2024-09-17T09:40:54.868 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:54 smithi078 ceph-mon[23689]: [17/Sep/2024:09:40:53] ENGINE Serving on https://172.21.15.78:7150 2024-09-17T09:40:54.868 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:54 smithi078 ceph-mon[23689]: [17/Sep/2024:09:40:53] ENGINE Bus STARTED 2024-09-17T09:40:54.868 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:54 smithi078 ceph-mon[23689]: [17/Sep/2024:09:40:53] ENGINE Error in HTTPServer.serve 2024-09-17T09:40:54.868 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Traceback (most recent call last): 2024-09-17T09:40:54.868 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T09:40:54.868 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self._connections.run(self.expiration_interval) 2024-09-17T09:40:54.868 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T09:40:54.868 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self._run(expiration_interval) 2024-09-17T09:40:54.868 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T09:40:54.868 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T09:40:54.868 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T09:40:54.868 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T09:40:54.869 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T09:40:54.869 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: s = self.context.wrap_socket( 2024-09-17T09:40:54.869 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T09:40:54.869 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: return self.sslsocket_class._create( 2024-09-17T09:40:54.869 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T09:40:54.869 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self.do_handshake() 2024-09-17T09:40:54.869 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T09:40:54.869 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self._sslobj.do_handshake() 2024-09-17T09:40:54.869 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T09:40:55.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:54 smithi136 ceph-mon[25953]: [17/Sep/2024:09:40:53] ENGINE Bus STARTING 2024-09-17T09:40:55.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:54 smithi136 ceph-mon[25953]: [17/Sep/2024:09:40:53] ENGINE Serving on http://172.21.15.78:8765 2024-09-17T09:40:55.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:54 smithi136 ceph-mon[25953]: [17/Sep/2024:09:40:53] ENGINE Serving on https://172.21.15.78:7150 2024-09-17T09:40:55.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:54 smithi136 ceph-mon[25953]: [17/Sep/2024:09:40:53] ENGINE Bus STARTED 2024-09-17T09:40:55.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:54 smithi136 ceph-mon[25953]: [17/Sep/2024:09:40:53] ENGINE Error in HTTPServer.serve 2024-09-17T09:40:55.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: Traceback (most recent call last): 2024-09-17T09:40:55.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T09:40:55.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: self._connections.run(self.expiration_interval) 2024-09-17T09:40:55.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T09:40:55.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: self._run(expiration_interval) 2024-09-17T09:40:55.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T09:40:55.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T09:40:55.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T09:40:55.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T09:40:55.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T09:40:55.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: s = self.context.wrap_socket( 2024-09-17T09:40:55.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T09:40:55.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: return self.sslsocket_class._create( 2024-09-17T09:40:55.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T09:40:55.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: self.do_handshake() 2024-09-17T09:40:55.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T09:40:55.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: self._sslobj.do_handshake() 2024-09-17T09:40:55.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T09:40:55.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 124417 12.46 MB/sec execute 110 sec latency 40.891 ms 2024-09-17T09:40:55.900 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:55 smithi078 ceph-mon[23689]: pgmap v5: 97 pgs: 97 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-17T09:40:55.900 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:55 smithi078 ceph-mon[23689]: mgrmap e33: smithi078.rsqofi(active, since 4s), standbys: smithi136.hicmlm 2024-09-17T09:40:55.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:55 smithi136 ceph-mon[25953]: pgmap v5: 97 pgs: 97 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-17T09:40:55.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:55 smithi136 ceph-mon[25953]: mgrmap e33: smithi078.rsqofi(active, since 4s), standbys: smithi136.hicmlm 2024-09-17T09:40:56.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 124698 12.38 MB/sec execute 111 sec latency 49.655 ms 2024-09-17T09:40:57.177 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:57 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:40:57.178 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:57 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:40:57.178 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:57 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:40:57.178 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:57 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:40:57.313 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:57 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:40:57.314 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:57 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:40:57.314 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:57 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:40:57.314 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:57 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:40:57.398 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-17T09:40:57.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 125019 12.36 MB/sec execute 112 sec latency 55.196 ms 2024-09-17T09:40:58.199 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:58 smithi078 ceph-mon[23689]: pgmap v6: 97 pgs: 97 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.4 MiB/s wr, 153 op/s 2024-09-17T09:40:58.200 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:58 smithi078 ceph-mon[23689]: Standby manager daemon smithi136.hicmlm restarted 2024-09-17T09:40:58.200 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:58 smithi078 ceph-mon[23689]: Standby manager daemon smithi136.hicmlm started 2024-09-17T09:40:58.200 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:58 smithi078 ceph-mon[23689]: from='mgr.? 172.21.15.136:0/2255135992' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi136.hicmlm/crt"}]: dispatch 2024-09-17T09:40:58.200 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:58 smithi078 ceph-mon[23689]: from='mgr.? 172.21.15.136:0/2255135992' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T09:40:58.200 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:58 smithi078 ceph-mon[23689]: from='mgr.? 172.21.15.136:0/2255135992' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi136.hicmlm/key"}]: dispatch 2024-09-17T09:40:58.200 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:58 smithi078 ceph-mon[23689]: from='mgr.? 172.21.15.136:0/2255135992' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T09:40:58.321 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:58 smithi136 ceph-mon[25953]: pgmap v6: 97 pgs: 97 active+clean; 329 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.4 MiB/s wr, 153 op/s 2024-09-17T09:40:58.321 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:58 smithi136 ceph-mon[25953]: Standby manager daemon smithi136.hicmlm restarted 2024-09-17T09:40:58.321 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:58 smithi136 ceph-mon[25953]: Standby manager daemon smithi136.hicmlm started 2024-09-17T09:40:58.321 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:58 smithi136 ceph-mon[25953]: from='mgr.? 172.21.15.136:0/2255135992' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi136.hicmlm/crt"}]: dispatch 2024-09-17T09:40:58.321 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:58 smithi136 ceph-mon[25953]: from='mgr.? 172.21.15.136:0/2255135992' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T09:40:58.322 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:58 smithi136 ceph-mon[25953]: from='mgr.? 172.21.15.136:0/2255135992' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi136.hicmlm/key"}]: dispatch 2024-09-17T09:40:58.322 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:58 smithi136 ceph-mon[25953]: from='mgr.? 172.21.15.136:0/2255135992' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T09:40:58.530 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:40:58.530 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (85s) 16s ago 10m 19.1M - 0.27.0 11f11916f8cd 97db2373ce92 2024-09-17T09:40:58.530 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (10m) 16s ago 10m 6400k - 17.2.0 e1d6a67b021e 46d44333624b 2024-09-17T09:40:58.530 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi136 smithi136 running (9m) 2s ago 9m 6395k - 17.2.0 e1d6a67b021e 56ce035cb482 2024-09-17T09:40:58.530 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (75s) 16s ago 10m 42.9M - dad864ee21e9 db74b0f8e0a7 2024-09-17T09:40:58.530 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.jdrsup smithi078 running (4m) 16s ago 4m 37.0M - 17.2.0 e1d6a67b021e 8f834e15ba88 2024-09-17T09:40:58.530 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi136.smslff smithi136 running (4m) 2s ago 4m 15.2M - 17.2.0 e1d6a67b021e 0c20e2b328d3 2024-09-17T09:40:58.530 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rsqofi smithi078 *:8443,9283,8765 running (28s) 16s ago 11m 86.6M - 19.3.0-4963-ge646f729 f271290a0950 7dda3108c9a0 2024-09-17T09:40:58.530 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi136.hicmlm smithi136 *:8443,9283 running (2m) 2s ago 9m 123M - 19.3.0-4963-ge646f729 f271290a0950 bbd092a7e4ca 2024-09-17T09:40:58.530 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (11m) 16s ago 11m 56.2M 2048M 17.2.0 e1d6a67b021e 09299430e248 2024-09-17T09:40:58.530 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi136 smithi136 running (9m) 2s ago 9m 44.8M 2048M 17.2.0 e1d6a67b021e 1bfdfb0617bb 2024-09-17T09:40:58.531 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.tcwbvi smithi078 *:2049 running (4m) 16s ago 4m 262M - 4.0 e1d6a67b021e 30c87cb46555 2024-09-17T09:40:58.531 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi136.regjvc smithi136 *:2049 running (4m) 2s ago 3m 85.9M - 4.0 e1d6a67b021e b88e4dc24376 2024-09-17T09:40:58.531 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (67s) 16s ago 10m 8228k - 1.7.0 72c9c2088986 52dad3adeeaa 2024-09-17T09:40:58.531 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi136 smithi136 *:9100 running (38s) 2s ago 9m 8032k - 1.7.0 72c9c2088986 f2bc5856e35a 2024-09-17T09:40:58.531 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (8m) 16s ago 8m 433M 1936M 17.2.0 e1d6a67b021e b4ca42f8477d 2024-09-17T09:40:58.531 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (8m) 16s ago 8m 327M 1936M 17.2.0 e1d6a67b021e 351873014de4 2024-09-17T09:40:58.531 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (7m) 16s ago 7m 260M 1936M 17.2.0 e1d6a67b021e 07eb71e4fa57 2024-09-17T09:40:58.531 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (7m) 16s ago 7m 391M 1936M 17.2.0 e1d6a67b021e 1b3a8be89881 2024-09-17T09:40:58.531 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi136 running (7m) 2s ago 7m 427M 2704M 17.2.0 e1d6a67b021e a0c5b4de2414 2024-09-17T09:40:58.531 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi136 running (6m) 2s ago 6m 466M 2704M 17.2.0 e1d6a67b021e 30a3049955f9 2024-09-17T09:40:58.531 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi136 running (6m) 2s ago 6m 407M 2704M 17.2.0 e1d6a67b021e 1698e1567c33 2024-09-17T09:40:58.531 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi136 running (5m) 2s ago 5m 541M 2704M 17.2.0 e1d6a67b021e 5d3b08cecd78 2024-09-17T09:40:58.531 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (43s) 16s ago 9m 39.6M - 2.51.0 1d3b7f56885b 65efbf64439f 2024-09-17T09:40:58.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 125655 12.36 MB/sec execute 113 sec latency 34.037 ms 2024-09-17T09:40:59.204 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:59 smithi078 ceph-mon[23689]: from='client.14802 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:40:59.205 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:59 smithi078 ceph-mon[23689]: from='client.14806 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:40:59.205 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:40:59 smithi078 ceph-mon[23689]: mgrmap e34: smithi078.rsqofi(active, since 7s), standbys: smithi136.hicmlm 2024-09-17T09:40:59.283 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:40:59.283 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-17T09:40:59.283 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:40:59.284 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:40:59.284 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-17T09:40:59.284 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:40:59.284 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:40:59.284 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-17T09:40:59.284 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T09:40:59.284 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:40:59.284 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-17T09:40:59.284 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:40:59.284 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:40:59.284 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-17T09:40:59.284 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-17T09:40:59.284 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:40:59.284 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-17T09:40:59.285 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:40:59.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:59 smithi136 ceph-mon[25953]: from='client.14802 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:40:59.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:59 smithi136 ceph-mon[25953]: from='client.14806 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:40:59.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:40:59 smithi136 ceph-mon[25953]: mgrmap e34: smithi078.rsqofi(active, since 7s), standbys: smithi136.hicmlm 2024-09-17T09:40:59.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 126341 12.38 MB/sec execute 114 sec latency 26.778 ms 2024-09-17T09:40:59.861 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:40:59.861 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407", 2024-09-17T09:40:59.862 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-17T09:40:59.862 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T09:40:59.862 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-17T09:40:59.862 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr" 2024-09-17T09:40:59.862 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-17T09:40:59.862 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "2/23 daemons upgraded", 2024-09-17T09:40:59.862 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "", 2024-09-17T09:40:59.862 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-17T09:40:59.862 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:41:00.208 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:00 smithi078 ceph-mon[23689]: from='client.24545 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:41:00.208 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:00 smithi078 ceph-mon[23689]: pgmap v7: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 30 KiB/s rd, 4.3 MiB/s wr, 376 op/s 2024-09-17T09:41:00.208 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:00 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/3909111607' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:41:00.393 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-17T09:41:00.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:00 smithi136 ceph-mon[25953]: from='client.24545 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:41:00.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:00 smithi136 ceph-mon[25953]: pgmap v7: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 30 KiB/s rd, 4.3 MiB/s wr, 376 op/s 2024-09-17T09:41:00.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:00 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/3909111607' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:41:00.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 126744 12.41 MB/sec execute 115 sec latency 37.729 ms 2024-09-17T09:41:01.234 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:01 smithi078 ceph-mon[23689]: from='client.24553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:41:01.235 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:01 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/4191363916' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:41:01.235 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:01 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:01.235 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:01 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:01.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:01 smithi136 ceph-mon[25953]: from='client.24553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:41:01.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:01 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/4191363916' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:41:01.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:01 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:01.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:01 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:01.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 127170 12.41 MB/sec execute 116 sec latency 39.831 ms 2024-09-17T09:41:02.269 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:02 smithi078 ceph-mon[23689]: pgmap v8: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 3.3 MiB/s wr, 292 op/s 2024-09-17T09:41:02.269 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:02 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:02.269 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:02 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:02.478 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:02 smithi136 ceph-mon[25953]: pgmap v8: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 3.3 MiB/s wr, 292 op/s 2024-09-17T09:41:02.478 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:02 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:02.478 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:02 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:02.787 INFO:teuthology.orchestra.run.smithi078.stdout: 5 127521 12.42 MB/sec execute 117 sec latency 64.190 ms 2024-09-17T09:41:03.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 127765 12.38 MB/sec execute 118 sec latency 56.403 ms 2024-09-17T09:41:04.316 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:04 smithi078 ceph-mon[23689]: pgmap v9: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 5.7 MiB/s wr, 559 op/s 2024-09-17T09:41:04.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:04 smithi136 ceph-mon[25953]: pgmap v9: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 5.7 MiB/s wr, 559 op/s 2024-09-17T09:41:04.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 128025 12.33 MB/sec execute 119 sec latency 56.951 ms 2024-09-17T09:41:05.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 128328 12.29 MB/sec execute 120 sec latency 49.865 ms 2024-09-17T09:41:06.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:06 smithi078 ceph-mon[23689]: pgmap v10: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 22 KiB/s rd, 5.2 MiB/s wr, 511 op/s 2024-09-17T09:41:06.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:06 smithi136 ceph-mon[25953]: pgmap v10: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 22 KiB/s rd, 5.2 MiB/s wr, 511 op/s 2024-09-17T09:41:06.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 128638 12.25 MB/sec execute 121 sec latency 56.350 ms 2024-09-17T09:41:07.383 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:07 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T09:41:07.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:07 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T09:41:07.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 128895 12.17 MB/sec execute 122 sec latency 49.258 ms 2024-09-17T09:41:08.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:08 smithi078 ceph-mon[23689]: pgmap v11: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 6.9 MiB/s wr, 669 op/s 2024-09-17T09:41:08.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:08 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:08.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:08 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:08.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:08 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:41:08.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:08 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:41:08.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:08 smithi136 ceph-mon[25953]: pgmap v11: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 6.9 MiB/s wr, 669 op/s 2024-09-17T09:41:08.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:08 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:08.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:08 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:08.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:08 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:41:08.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:08 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:41:08.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 129238 12.14 MB/sec execute 123 sec latency 57.461 ms 2024-09-17T09:41:09.178 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:09 smithi078 ceph-mon[23689]: Updating smithi078:/etc/ceph/ceph.conf 2024-09-17T09:41:09.179 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:09 smithi078 ceph-mon[23689]: Updating smithi136:/etc/ceph/ceph.conf 2024-09-17T09:41:09.179 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:09 smithi078 ceph-mon[23689]: Updating smithi078:/var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T09:41:09.179 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:09 smithi078 ceph-mon[23689]: Updating smithi136:/var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T09:41:09.179 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:09 smithi078 ceph-mon[23689]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-09-17T09:41:09.179 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:09 smithi078 ceph-mon[23689]: Updating smithi136:/etc/ceph/ceph.client.admin.keyring 2024-09-17T09:41:09.179 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:09 smithi078 ceph-mon[23689]: Updating smithi078:/var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T09:41:09.179 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:09 smithi078 ceph-mon[23689]: Updating smithi136:/var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T09:41:09.179 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:09 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:09.179 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:09 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:09.179 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:09 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:09.179 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:09 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:09.179 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:09 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:09.179 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:09 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:09.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:09 smithi136 ceph-mon[25953]: Updating smithi078:/etc/ceph/ceph.conf 2024-09-17T09:41:09.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:09 smithi136 ceph-mon[25953]: Updating smithi136:/etc/ceph/ceph.conf 2024-09-17T09:41:09.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:09 smithi136 ceph-mon[25953]: Updating smithi078:/var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T09:41:09.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:09 smithi136 ceph-mon[25953]: Updating smithi136:/var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T09:41:09.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:09 smithi136 ceph-mon[25953]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-09-17T09:41:09.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:09 smithi136 ceph-mon[25953]: Updating smithi136:/etc/ceph/ceph.client.admin.keyring 2024-09-17T09:41:09.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:09 smithi136 ceph-mon[25953]: Updating smithi078:/var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T09:41:09.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:09 smithi136 ceph-mon[25953]: Updating smithi136:/var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T09:41:09.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:09 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:09.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:09 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:09.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:09 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:09.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:09 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:09.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:09 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:09.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:09 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:09.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 129638 12.09 MB/sec execute 124 sec latency 55.454 ms 2024-09-17T09:41:10.227 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:10 smithi078 ceph-mon[23689]: Reconfiguring prometheus.smithi078 (dependencies changed)... 2024-09-17T09:41:10.227 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:10 smithi078 ceph-mon[23689]: Reconfiguring daemon prometheus.smithi078 on smithi078 2024-09-17T09:41:10.227 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:10 smithi078 ceph-mon[23689]: pgmap v12: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 22 KiB/s rd, 6.0 MiB/s wr, 646 op/s 2024-09-17T09:41:10.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:10 smithi136 ceph-mon[25953]: Reconfiguring prometheus.smithi078 (dependencies changed)... 2024-09-17T09:41:10.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:10 smithi136 ceph-mon[25953]: Reconfiguring daemon prometheus.smithi078 on smithi078 2024-09-17T09:41:10.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:10 smithi136 ceph-mon[25953]: pgmap v12: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 22 KiB/s rd, 6.0 MiB/s wr, 646 op/s 2024-09-17T09:41:10.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 130049 12.09 MB/sec execute 125 sec latency 27.384 ms 2024-09-17T09:41:11.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 130590 12.15 MB/sec execute 126 sec latency 60.040 ms 2024-09-17T09:41:12.303 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:12 smithi078 ceph-mon[23689]: pgmap v13: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.0 MiB/s wr, 491 op/s 2024-09-17T09:41:12.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:12 smithi136 ceph-mon[25953]: pgmap v13: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.0 MiB/s wr, 491 op/s 2024-09-17T09:41:12.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 131067 12.19 MB/sec execute 127 sec latency 39.771 ms 2024-09-17T09:41:13.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 131284 12.16 MB/sec execute 128 sec latency 60.787 ms 2024-09-17T09:41:14.368 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:14 smithi078 ceph-mon[23689]: pgmap v14: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 720 op/s 2024-09-17T09:41:14.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:14 smithi136 ceph-mon[25953]: pgmap v14: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 720 op/s 2024-09-17T09:41:14.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 131528 12.12 MB/sec execute 129 sec latency 77.544 ms 2024-09-17T09:41:15.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 131821 12.06 MB/sec execute 130 sec latency 53.097 ms 2024-09-17T09:41:16.406 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:16 smithi078 ceph-mon[23689]: pgmap v15: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.3 MiB/s wr, 427 op/s 2024-09-17T09:41:16.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:16 smithi136 ceph-mon[25953]: pgmap v15: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.3 MiB/s wr, 427 op/s 2024-09-17T09:41:16.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 132221 12.05 MB/sec execute 131 sec latency 41.905 ms 2024-09-17T09:41:17.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 132630 12.03 MB/sec execute 132 sec latency 59.476 ms 2024-09-17T09:41:18.407 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:18 smithi136 ceph-mon[25953]: pgmap v16: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.1 MiB/s wr, 580 op/s 2024-09-17T09:41:18.407 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:18 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:18.407 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:18 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:18.407 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:18 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:41:18.407 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:18 smithi136 ceph-mon[25953]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:41:18.407 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:18 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:41:18.464 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:18 smithi078 ceph-mon[23689]: pgmap v16: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.1 MiB/s wr, 580 op/s 2024-09-17T09:41:18.464 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:18 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:18.464 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:18 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:18.465 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:18 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:41:18.465 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:18 smithi078 ceph-mon[23689]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T09:41:18.465 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:18 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:41:18.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 132994 11.98 MB/sec execute 133 sec latency 44.844 ms 2024-09-17T09:41:19.756 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:19 smithi078 ceph-mon[23689]: pgmap v17: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.0 MiB/s wr, 479 op/s 2024-09-17T09:41:19.757 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:19 smithi078 ceph-mon[23689]: Upgrade: Updating mgr.smithi136.hicmlm 2024-09-17T09:41:19.757 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:19 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi136.hicmlm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T09:41:19.757 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:19 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T09:41:19.757 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:19 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:41:19.757 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:19 smithi078 ceph-mon[23689]: Deploying daemon mgr.smithi136.hicmlm on smithi136 2024-09-17T09:41:19.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 133427 11.96 MB/sec execute 134 sec latency 42.807 ms 2024-09-17T09:41:19.850 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:19 smithi136 ceph-mon[25953]: pgmap v17: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.0 MiB/s wr, 479 op/s 2024-09-17T09:41:19.850 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:19 smithi136 ceph-mon[25953]: Upgrade: Updating mgr.smithi136.hicmlm 2024-09-17T09:41:19.850 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:19 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi136.hicmlm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T09:41:19.850 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:19 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T09:41:19.851 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:19 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:41:19.851 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:19 smithi136 ceph-mon[25953]: Deploying daemon mgr.smithi136.hicmlm on smithi136 2024-09-17T09:41:20.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 133751 11.94 MB/sec execute 135 sec latency 40.767 ms 2024-09-17T09:41:21.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 134213 11.99 MB/sec execute 136 sec latency 60.148 ms 2024-09-17T09:41:21.916 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:21 smithi078 ceph-mon[23689]: pgmap v18: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.4 MiB/s wr, 438 op/s 2024-09-17T09:41:21.917 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:21 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T09:41:21.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:21 smithi136 ceph-mon[25953]: pgmap v18: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.4 MiB/s wr, 438 op/s 2024-09-17T09:41:21.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:21 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T09:41:22.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 134539 11.97 MB/sec execute 137 sec latency 44.360 ms 2024-09-17T09:41:23.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 134760 11.96 MB/sec execute 138 sec latency 57.149 ms 2024-09-17T09:41:23.971 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:23 smithi078 ceph-mon[23689]: pgmap v19: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.5 MiB/s wr, 669 op/s 2024-09-17T09:41:23.971 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:23 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:23.971 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:23 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:23.971 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:23 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:41:24.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:23 smithi136 ceph-mon[25953]: pgmap v19: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.5 MiB/s wr, 669 op/s 2024-09-17T09:41:24.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:23 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:24.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:23 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:24.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:23 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:41:24.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 134925 11.89 MB/sec execute 139 sec latency 111.142 ms 2024-09-17T09:41:25.004 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:24 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:25.004 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:24 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:25.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:24 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:25.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:24 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:25.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 135114 11.85 MB/sec execute 140 sec latency 54.499 ms 2024-09-17T09:41:26.042 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:25 smithi078 ceph-mon[23689]: pgmap v20: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.4 MiB/s wr, 440 op/s 2024-09-17T09:41:26.220 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:25 smithi136 ceph-mon[25953]: pgmap v20: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.4 MiB/s wr, 440 op/s 2024-09-17T09:41:26.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 135478 11.82 MB/sec execute 141 sec latency 63.643 ms 2024-09-17T09:41:27.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 135846 11.79 MB/sec execute 142 sec latency 43.152 ms 2024-09-17T09:41:27.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:27 smithi136 ceph-mon[25953]: pgmap v21: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.9 MiB/s wr, 583 op/s 2024-09-17T09:41:28.089 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:27 smithi078 ceph-mon[23689]: pgmap v21: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.9 MiB/s wr, 583 op/s 2024-09-17T09:41:28.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 136211 11.78 MB/sec execute 143 sec latency 48.680 ms 2024-09-17T09:41:29.124 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:28 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:29.124 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:28 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:29.125 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:28 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:29.125 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:28 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:29.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:28 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:29.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:28 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:29.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:28 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:29.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:28 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:29.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 136522 11.73 MB/sec execute 144 sec latency 99.987 ms 2024-09-17T09:41:29.988 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:29 smithi136 ceph-mon[25953]: pgmap v22: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.5 MiB/s wr, 448 op/s 2024-09-17T09:41:30.138 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:29 smithi078 ceph-mon[23689]: pgmap v22: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.5 MiB/s wr, 448 op/s 2024-09-17T09:41:30.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 136926 11.70 MB/sec execute 145 sec latency 63.349 ms 2024-09-17T09:41:30.934 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-17T09:41:31.163 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:30 smithi078 ceph-mon[23689]: Standby manager daemon smithi136.hicmlm restarted 2024-09-17T09:41:31.163 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:30 smithi078 ceph-mon[23689]: Standby manager daemon smithi136.hicmlm started 2024-09-17T09:41:31.163 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:30 smithi078 ceph-mon[23689]: from='mgr.? 172.21.15.136:0/2495017980' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi136.hicmlm/crt"}]: dispatch 2024-09-17T09:41:31.164 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:30 smithi078 ceph-mon[23689]: from='mgr.? 172.21.15.136:0/2495017980' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T09:41:31.164 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:30 smithi078 ceph-mon[23689]: from='mgr.? 172.21.15.136:0/2495017980' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi136.hicmlm/key"}]: dispatch 2024-09-17T09:41:31.164 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:30 smithi078 ceph-mon[23689]: from='mgr.? 172.21.15.136:0/2495017980' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T09:41:31.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:30 smithi136 ceph-mon[25953]: Standby manager daemon smithi136.hicmlm restarted 2024-09-17T09:41:31.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:30 smithi136 ceph-mon[25953]: Standby manager daemon smithi136.hicmlm started 2024-09-17T09:41:31.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:30 smithi136 ceph-mon[25953]: from='mgr.? 172.21.15.136:0/2495017980' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi136.hicmlm/crt"}]: dispatch 2024-09-17T09:41:31.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:30 smithi136 ceph-mon[25953]: from='mgr.? 172.21.15.136:0/2495017980' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T09:41:31.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:30 smithi136 ceph-mon[25953]: from='mgr.? 172.21.15.136:0/2495017980' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi136.hicmlm/key"}]: dispatch 2024-09-17T09:41:31.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:30 smithi136 ceph-mon[25953]: from='mgr.? 172.21.15.136:0/2495017980' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T09:41:31.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 137175 11.66 MB/sec execute 146 sec latency 53.357 ms 2024-09-17T09:41:31.840 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:41:31.840 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (118s) 0s ago 11m 19.1M - 0.27.0 11f11916f8cd 97db2373ce92 2024-09-17T09:41:31.840 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (10m) 0s ago 10m 6400k - 17.2.0 e1d6a67b021e 46d44333624b 2024-09-17T09:41:31.841 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi136 smithi136 running (10m) 3s ago 10m 6395k - 17.2.0 e1d6a67b021e 56ce035cb482 2024-09-17T09:41:31.841 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (109s) 0s ago 10m 53.0M - dad864ee21e9 db74b0f8e0a7 2024-09-17T09:41:31.841 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.jdrsup smithi078 running (4m) 0s ago 4m 37.2M - 17.2.0 e1d6a67b021e 8f834e15ba88 2024-09-17T09:41:31.841 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi136.smslff smithi136 running (4m) 3s ago 4m 15.4M - 17.2.0 e1d6a67b021e 0c20e2b328d3 2024-09-17T09:41:31.841 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rsqofi smithi078 *:8443,9283,8765 running (62s) 0s ago 12m 508M - 19.3.0-4963-ge646f729 f271290a0950 7dda3108c9a0 2024-09-17T09:41:31.841 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi136.hicmlm smithi136 *:8443,9283,8765 running (8s) 3s ago 10m 86.3M - 19.3.0-4963-ge646f729 f271290a0950 fc518b58e07a 2024-09-17T09:41:31.841 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (12m) 0s ago 12m 59.7M 2048M 17.2.0 e1d6a67b021e 09299430e248 2024-09-17T09:41:31.841 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi136 smithi136 running (10m) 3s ago 10m 44.3M 2048M 17.2.0 e1d6a67b021e 1bfdfb0617bb 2024-09-17T09:41:31.841 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.tcwbvi smithi078 *:2049 running (4m) 0s ago 4m 265M - 4.0 e1d6a67b021e 30c87cb46555 2024-09-17T09:41:31.841 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi136.regjvc smithi136 *:2049 running (4m) 3s ago 4m 87.5M - 4.0 e1d6a67b021e b88e4dc24376 2024-09-17T09:41:31.841 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (100s) 0s ago 10m 9663k - 1.7.0 72c9c2088986 52dad3adeeaa 2024-09-17T09:41:31.841 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi136 smithi136 *:9100 running (71s) 3s ago 10m 12.2M - 1.7.0 72c9c2088986 f2bc5856e35a 2024-09-17T09:41:31.841 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (9m) 0s ago 9m 478M 1936M 17.2.0 e1d6a67b021e b4ca42f8477d 2024-09-17T09:41:31.841 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (8m) 0s ago 8m 407M 1936M 17.2.0 e1d6a67b021e 351873014de4 2024-09-17T09:41:31.842 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (8m) 0s ago 8m 276M 1936M 17.2.0 e1d6a67b021e 07eb71e4fa57 2024-09-17T09:41:31.842 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (8m) 0s ago 8m 440M 1936M 17.2.0 e1d6a67b021e 1b3a8be89881 2024-09-17T09:41:31.842 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi136 running (7m) 3s ago 7m 480M 2704M 17.2.0 e1d6a67b021e a0c5b4de2414 2024-09-17T09:41:31.842 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi136 running (7m) 3s ago 7m 515M 2704M 17.2.0 e1d6a67b021e 30a3049955f9 2024-09-17T09:41:31.842 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi136 running (6m) 3s ago 6m 430M 2704M 17.2.0 e1d6a67b021e 1698e1567c33 2024-09-17T09:41:31.842 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi136 running (6m) 3s ago 6m 343M 2704M 17.2.0 e1d6a67b021e 5d3b08cecd78 2024-09-17T09:41:31.842 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (15s) 0s ago 9m 41.7M - 2.51.0 1d3b7f56885b 6e5e8c721eb0 2024-09-17T09:41:31.979 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:31 smithi078 ceph-mon[23689]: pgmap v23: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 392 op/s 2024-09-17T09:41:31.979 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:31 smithi078 ceph-mon[23689]: from='client.14828 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:41:31.980 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:31 smithi078 ceph-mon[23689]: mgrmap e35: smithi078.rsqofi(active, since 40s), standbys: smithi136.hicmlm 2024-09-17T09:41:31.980 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:31 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:31.980 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:31 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:32.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:31 smithi136 ceph-mon[25953]: pgmap v23: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 392 op/s 2024-09-17T09:41:32.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:31 smithi136 ceph-mon[25953]: from='client.14828 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:41:32.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:31 smithi136 ceph-mon[25953]: mgrmap e35: smithi078.rsqofi(active, since 40s), standbys: smithi136.hicmlm 2024-09-17T09:41:32.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:31 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:32.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:31 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:32.405 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:41:32.406 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-17T09:41:32.406 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:41:32.406 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:41:32.406 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-17T09:41:32.406 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:41:32.406 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:41:32.406 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-17T09:41:32.406 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T09:41:32.406 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:41:32.406 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-17T09:41:32.407 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:41:32.407 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:41:32.407 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-17T09:41:32.407 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-17T09:41:32.407 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:41:32.407 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-17T09:41:32.407 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:41:32.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 137548 11.70 MB/sec execute 147 sec latency 46.813 ms 2024-09-17T09:41:32.900 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:41:32.901 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407", 2024-09-17T09:41:32.901 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-17T09:41:32.901 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T09:41:32.901 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-17T09:41:32.901 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr" 2024-09-17T09:41:32.901 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-17T09:41:32.901 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "2/23 daemons upgraded", 2024-09-17T09:41:32.901 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading mgr daemons", 2024-09-17T09:41:32.901 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-17T09:41:32.901 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:41:32.996 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:32 smithi078 ceph-mon[23689]: from='client.24569 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:41:32.997 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:32 smithi078 ceph-mon[23689]: from='client.14836 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:41:32.997 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:32 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/352579486' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:41:32.997 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:32 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:32.997 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:32 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:32.997 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:32 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:41:32.997 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:32 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:41:32.997 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:32 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:32.997 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:32 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:33.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:32 smithi136 ceph-mon[25953]: from='client.24569 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:41:33.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:32 smithi136 ceph-mon[25953]: from='client.14836 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:41:33.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:32 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/352579486' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:41:33.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:32 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:33.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:32 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:33.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:32 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:41:33.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:32 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T09:41:33.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:32 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:33.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:32 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:33.639 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-17T09:41:33.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 137919 11.70 MB/sec execute 148 sec latency 50.318 ms 2024-09-17T09:41:34.007 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:33 smithi078 ceph-mon[23689]: pgmap v24: 97 pgs: 97 active+clean; 327 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.9 MiB/s wr, 607 op/s 2024-09-17T09:41:34.007 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:33 smithi078 ceph-mon[23689]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:41:34.007 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:33 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:41:34.007 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:33 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:41:34.007 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:33 smithi078 ceph-mon[23689]: Upgrade: Setting container_image for all mgr 2024-09-17T09:41:34.007 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:33 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:34.007 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:33 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi078.rsqofi"}]: dispatch 2024-09-17T09:41:34.007 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:33 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi078.rsqofi"}]': finished 2024-09-17T09:41:34.007 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:33 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi136.hicmlm"}]: dispatch 2024-09-17T09:41:34.007 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:33 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi136.hicmlm"}]': finished 2024-09-17T09:41:34.008 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:33 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T09:41:34.008 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:33 smithi078 ceph-mon[23689]: from='client.? 172.21.15.78:0/309111704' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:41:34.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:33 smithi136 ceph-mon[25953]: pgmap v24: 97 pgs: 97 active+clean; 327 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.9 MiB/s wr, 607 op/s 2024-09-17T09:41:34.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:33 smithi136 ceph-mon[25953]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:41:34.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:33 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T09:41:34.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:33 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T09:41:34.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:33 smithi136 ceph-mon[25953]: Upgrade: Setting container_image for all mgr 2024-09-17T09:41:34.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:33 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:34.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:33 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi078.rsqofi"}]: dispatch 2024-09-17T09:41:34.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:33 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi078.rsqofi"}]': finished 2024-09-17T09:41:34.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:33 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi136.hicmlm"}]: dispatch 2024-09-17T09:41:34.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:33 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi136.hicmlm"}]': finished 2024-09-17T09:41:34.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:33 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T09:41:34.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:33 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/309111704' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T09:41:34.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 138274 11.69 MB/sec execute 149 sec latency 51.168 ms 2024-09-17T09:41:35.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 138586 11.69 MB/sec execute 150 sec latency 41.427 ms 2024-09-17T09:41:36.046 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:35 smithi078 ceph-mon[23689]: pgmap v25: 97 pgs: 97 active+clean; 327 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.7 MiB/s wr, 376 op/s 2024-09-17T09:41:36.047 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:35 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:36.047 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:35 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T09:41:36.047 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:35 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T09:41:36.047 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:35 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:41:36.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:35 smithi136 ceph-mon[25953]: pgmap v25: 97 pgs: 97 active+clean; 327 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.7 MiB/s wr, 376 op/s 2024-09-17T09:41:36.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:35 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:36.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:35 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T09:41:36.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:35 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T09:41:36.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:35 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T09:41:36.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 139175 11.69 MB/sec execute 151 sec latency 53.499 ms 2024-09-17T09:41:37.050 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:36 smithi078 ceph-mon[23689]: Upgrade: Updating mon.smithi078 2024-09-17T09:41:37.050 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:36 smithi078 ceph-mon[23689]: Deploying daemon mon.smithi078 on smithi078 2024-09-17T09:41:37.051 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:36 smithi078 ceph-mon[23689]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T09:41:37.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:36 smithi136 ceph-mon[25953]: Upgrade: Updating mon.smithi078 2024-09-17T09:41:37.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:36 smithi136 ceph-mon[25953]: Deploying daemon mon.smithi078 on smithi078 2024-09-17T09:41:37.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:36 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T09:41:37.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 139792 11.71 MB/sec execute 152 sec latency 35.358 ms 2024-09-17T09:41:38.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:37 smithi078 ceph-mon[23689]: pgmap v26: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.9 MiB/s wr, 589 op/s 2024-09-17T09:41:38.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:37 smithi136 ceph-mon[25953]: pgmap v26: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.9 MiB/s wr, 589 op/s 2024-09-17T09:41:38.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 140553 11.73 MB/sec execute 153 sec latency 21.553 ms 2024-09-17T09:41:38.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:38 smithi078 systemd[1]: Stopping Ceph mon.smithi078 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:41:39.073 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:39 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi078[23685]: 2024-09-17T09:41:39.009+0000 7ff8e4e2a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi078 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:41:39.073 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:39 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi078[23685]: 2024-09-17T09:41:39.009+0000 7ff8e4e2a700 -1 mon.smithi078@0(leader) e2 *** Got Signal Terminated *** 2024-09-17T09:41:39.328 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:39 smithi078 podman[63626]: 2024-09-17 09:41:39.091909891 +0000 UTC m=+0.251415318 container died 09299430e248d2be74bcfcf04c8960ca42faf745d77f0fb3c9294a8dc0d7f44a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi078, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, io.buildah.version=1.19.8, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-17T09:41:39.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 141018 11.75 MB/sec execute 154 sec latency 51.587 ms 2024-09-17T09:41:40.333 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:40 smithi078 podman[63626]: 2024-09-17 09:41:40.214167514 +0000 UTC m=+1.373672945 container cleanup 09299430e248d2be74bcfcf04c8960ca42faf745d77f0fb3c9294a8dc0d7f44a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi078, io.k8s.display-name=CentOS Stream 8, release=754, RELEASE=HEAD, name=centos-stream, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, architecture=x86_64, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream) 2024-09-17T09:41:40.334 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:40 smithi078 bash[63626]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi078 2024-09-17T09:41:40.584 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:40 smithi078 podman[63637]: 2024-09-17 09:41:40.566702378 +0000 UTC m=+1.466564916 container remove 09299430e248d2be74bcfcf04c8960ca42faf745d77f0fb3c9294a8dc0d7f44a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi078, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, vendor=Red Hat, Inc., release=754, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, distribution-scope=public, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, version=8, name=centos-stream, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container) 2024-09-17T09:41:40.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 141623 11.80 MB/sec execute 155 sec latency 49.976 ms 2024-09-17T09:41:41.087 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:41 smithi078 systemd[1]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@mon.smithi078.service: Deactivated successfully. 2024-09-17T09:41:41.087 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:41 smithi078 systemd[1]: Stopped Ceph mon.smithi078 for 35c0cd22-74d7-11ef-bceb-c7b262605968. 2024-09-17T09:41:41.087 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:41 smithi078 systemd[1]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@mon.smithi078.service: Consumed 16.344s CPU time. 2024-09-17T09:41:41.592 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:41 smithi078 systemd[1]: Starting Ceph mon.smithi078 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:41:41.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 142215 11.86 MB/sec execute 156 sec latency 34.969 ms 2024-09-17T09:41:41.844 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:41 smithi078 podman[63768]: 2024-09-17 09:41:41.592833875 +0000 UTC m=+0.028514550 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:41:41.845 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:41 smithi078 podman[63768]: 2024-09-17 09:41:41.823948135 +0000 UTC m=+0.259628804 container create 3ead101d3db1d111420cb146309cd04437cd607a750073cf14fdb72f34049737 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi078, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-17T09:41:42.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 podman[63768]: 2024-09-17 09:41:42.311781484 +0000 UTC m=+0.747462168 container init 3ead101d3db1d111420cb146309cd04437cd607a750073cf14fdb72f34049737 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi078, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-17T09:41:42.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 podman[63768]: 2024-09-17 09:41:42.320947248 +0000 UTC m=+0.756627910 container start 3ead101d3db1d111420cb146309cd04437cd607a750073cf14fdb72f34049737 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi078, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-17T09:41:42.598 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: set uid:gid to 167:167 (ceph:ceph) 2024-09-17T09:41:42.598 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev), process ceph-mon, pid 2 2024-09-17T09:41:42.599 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: pidfile_write: ignore empty --pid-file 2024-09-17T09:41:42.599 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: load: jerasure load: lrc 2024-09-17T09:41:42.599 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: RocksDB version: 7.9.2 2024-09-17T09:41:42.599 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Git sha 0 2024-09-17T09:41:42.599 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Compile date 2024-09-16 20:37:15 2024-09-17T09:41:42.599 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: DB SUMMARY 2024-09-17T09:41:42.599 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: DB Session ID: 2DO59TOJUXI6E6S6UCXJ 2024-09-17T09:41:42.599 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: CURRENT file: CURRENT 2024-09-17T09:41:42.599 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: IDENTITY file: IDENTITY 2024-09-17T09:41:42.599 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: MANIFEST file: MANIFEST-000015 size: 979 Bytes 2024-09-17T09:41:42.599 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi078/store.db dir, Total Num: 1, files: 000030.sst 2024-09-17T09:41:42.600 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi078/store.db: 000028.log size: 2867604 ; 2024-09-17T09:41:42.600 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.error_if_exists: 0 2024-09-17T09:41:42.600 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.create_if_missing: 0 2024-09-17T09:41:42.600 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.paranoid_checks: 1 2024-09-17T09:41:42.600 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-17T09:41:42.600 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-17T09:41:42.600 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-17T09:41:42.600 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.env: 0x55f52f368120 2024-09-17T09:41:42.600 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.fs: PosixFileSystem 2024-09-17T09:41:42.600 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.info_log: 0x55f52feb18e0 2024-09-17T09:41:42.600 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.max_file_opening_threads: 16 2024-09-17T09:41:42.600 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.statistics: (nil) 2024-09-17T09:41:42.600 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.use_fsync: 0 2024-09-17T09:41:42.601 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.max_log_file_size: 0 2024-09-17T09:41:42.601 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-17T09:41:42.601 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-17T09:41:42.601 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.keep_log_file_num: 1000 2024-09-17T09:41:42.601 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.recycle_log_file_num: 0 2024-09-17T09:41:42.601 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.allow_fallocate: 1 2024-09-17T09:41:42.601 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.allow_mmap_reads: 0 2024-09-17T09:41:42.601 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.allow_mmap_writes: 0 2024-09-17T09:41:42.601 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.use_direct_reads: 0 2024-09-17T09:41:42.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-17T09:41:42.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.create_missing_column_families: 0 2024-09-17T09:41:42.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.db_log_dir: 2024-09-17T09:41:42.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.wal_dir: 2024-09-17T09:41:42.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-17T09:41:42.603 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-17T09:41:42.603 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-17T09:41:42.603 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-17T09:41:42.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-17T09:41:42.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-17T09:41:42.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.advise_random_on_open: 1 2024-09-17T09:41:42.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.db_write_buffer_size: 0 2024-09-17T09:41:42.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.write_buffer_manager: 0x55f52feb5b80 2024-09-17T09:41:42.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-17T09:41:42.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-17T09:41:42.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-17T09:41:42.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.rate_limiter: (nil) 2024-09-17T09:41:42.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-17T09:41:42.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.wal_recovery_mode: 2 2024-09-17T09:41:42.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.enable_thread_tracking: 0 2024-09-17T09:41:42.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.enable_pipelined_write: 0 2024-09-17T09:41:42.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.unordered_write: 0 2024-09-17T09:41:42.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-17T09:41:42.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-17T09:41:42.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-17T09:41:42.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-17T09:41:42.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.row_cache: None 2024-09-17T09:41:42.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.wal_filter: None 2024-09-17T09:41:42.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-17T09:41:42.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.allow_ingest_behind: 0 2024-09-17T09:41:42.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.two_write_queues: 0 2024-09-17T09:41:42.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.manual_wal_flush: 0 2024-09-17T09:41:42.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.wal_compression: 0 2024-09-17T09:41:42.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.atomic_flush: 0 2024-09-17T09:41:42.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-17T09:41:42.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-17T09:41:42.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-17T09:41:42.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.log_readahead_size: 0 2024-09-17T09:41:42.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-17T09:41:42.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.best_efforts_recovery: 0 2024-09-17T09:41:42.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-17T09:41:42.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-17T09:41:42.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.allow_data_in_errors: 0 2024-09-17T09:41:42.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.db_host_id: __hostname__ 2024-09-17T09:41:42.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-17T09:41:42.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.max_background_jobs: 2 2024-09-17T09:41:42.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.max_background_compactions: -1 2024-09-17T09:41:42.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.max_subcompactions: 1 2024-09-17T09:41:42.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-17T09:41:42.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-17T09:41:42.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-17T09:41:42.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.max_total_wal_size: 0 2024-09-17T09:41:42.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-17T09:41:42.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-17T09:41:42.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-17T09:41:42.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-17T09:41:42.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.max_open_files: -1 2024-09-17T09:41:42.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.bytes_per_sync: 0 2024-09-17T09:41:42.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-17T09:41:42.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-17T09:41:42.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.compaction_readahead_size: 0 2024-09-17T09:41:42.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.max_background_flushes: -1 2024-09-17T09:41:42.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Compression algorithms supported: 2024-09-17T09:41:42.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: kZSTD supported: 0 2024-09-17T09:41:42.609 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: kXpressCompression supported: 0 2024-09-17T09:41:42.609 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: kBZip2Compression supported: 0 2024-09-17T09:41:42.609 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-17T09:41:42.609 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: kLZ4Compression supported: 1 2024-09-17T09:41:42.609 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: kZlibCompression supported: 1 2024-09-17T09:41:42.609 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: kLZ4HCCompression supported: 1 2024-09-17T09:41:42.609 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: kSnappyCompression supported: 1 2024-09-17T09:41:42.609 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-17T09:41:42.609 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-17T09:41:42.609 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi078/store.db/MANIFEST-000015 2024-09-17T09:41:42.609 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-17T09:41:42.609 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-17T09:41:42.610 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.merge_operator: 2024-09-17T09:41:42.610 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.compaction_filter: None 2024-09-17T09:41:42.610 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.compaction_filter_factory: None 2024-09-17T09:41:42.610 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.sst_partitioner_factory: None 2024-09-17T09:41:42.610 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-17T09:41:42.610 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-17T09:41:42.610 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f52feb1540) 2024-09-17T09:41:42.610 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: cache_index_and_filter_blocks: 1 2024-09-17T09:41:42.610 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-17T09:41:42.610 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-17T09:41:42.610 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: pin_top_level_index_and_filter: 1 2024-09-17T09:41:42.610 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: index_type: 0 2024-09-17T09:41:42.611 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: data_block_index_type: 0 2024-09-17T09:41:42.611 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: index_shortening: 1 2024-09-17T09:41:42.611 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-17T09:41:42.611 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: checksum: 4 2024-09-17T09:41:42.611 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: no_block_cache: 0 2024-09-17T09:41:42.611 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache: 0x55f52fea5610 2024-09-17T09:41:42.611 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache_name: BinnedLRUCache 2024-09-17T09:41:42.611 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache_options: 2024-09-17T09:41:42.611 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: capacity : 536870912 2024-09-17T09:41:42.611 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: num_shard_bits : 4 2024-09-17T09:41:42.611 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: strict_capacity_limit : 0 2024-09-17T09:41:42.611 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: high_pri_pool_ratio: 0.000 2024-09-17T09:41:42.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache_compressed: (nil) 2024-09-17T09:41:42.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: persistent_cache: (nil) 2024-09-17T09:41:42.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_size: 4096 2024-09-17T09:41:42.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_size_deviation: 10 2024-09-17T09:41:42.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_restart_interval: 16 2024-09-17T09:41:42.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: index_block_restart_interval: 1 2024-09-17T09:41:42.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: metadata_block_size: 4096 2024-09-17T09:41:42.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: partition_filters: 0 2024-09-17T09:41:42.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: use_delta_encoding: 1 2024-09-17T09:41:42.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: filter_policy: bloomfilter 2024-09-17T09:41:42.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: whole_key_filtering: 1 2024-09-17T09:41:42.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: verify_compression: 0 2024-09-17T09:41:42.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: read_amp_bytes_per_bit: 0 2024-09-17T09:41:42.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: format_version: 5 2024-09-17T09:41:42.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: enable_index_compression: 1 2024-09-17T09:41:42.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_align: 0 2024-09-17T09:41:42.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: max_auto_readahead_size: 262144 2024-09-17T09:41:42.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: prepopulate_block_cache: 0 2024-09-17T09:41:42.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: initial_auto_readahead_size: 8192 2024-09-17T09:41:42.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: num_file_reads_for_auto_readahead: 2 2024-09-17T09:41:42.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.write_buffer_size: 33554432 2024-09-17T09:41:42.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.max_write_buffer_number: 2 2024-09-17T09:41:42.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.compression: NoCompression 2024-09-17T09:41:42.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.bottommost_compression: Disabled 2024-09-17T09:41:42.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.prefix_extractor: nullptr 2024-09-17T09:41:42.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-17T09:41:42.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.num_levels: 7 2024-09-17T09:41:42.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-17T09:41:42.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-17T09:41:42.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-17T09:41:42.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-17T09:41:42.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-17T09:41:42.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-17T09:41:42.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-17T09:41:42.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-17T09:41:42.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-17T09:41:42.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-17T09:41:42.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-17T09:41:42.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-17T09:41:42.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-17T09:41:42.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.compression_opts.level: 32767 2024-09-17T09:41:42.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.compression_opts.strategy: 0 2024-09-17T09:41:42.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-17T09:41:42.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-17T09:41:42.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-17T09:41:42.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-17T09:41:42.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.compression_opts.enabled: false 2024-09-17T09:41:42.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-17T09:41:42.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-17T09:41:42.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-17T09:41:42.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-17T09:41:42.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.target_file_size_base: 67108864 2024-09-17T09:41:42.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-17T09:41:42.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-17T09:41:42.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-17T09:41:42.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-17T09:41:42.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-17T09:41:42.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-17T09:41:42.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-17T09:41:42.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-17T09:41:42.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-17T09:41:42.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-17T09:41:42.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-17T09:41:42.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-17T09:41:42.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-17T09:41:42.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-17T09:41:42.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.arena_block_size: 1048576 2024-09-17T09:41:42.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-17T09:41:42.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-17T09:41:42.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.disable_auto_compactions: 0 2024-09-17T09:41:42.618 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-17T09:41:42.618 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-17T09:41:42.618 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-17T09:41:42.618 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-17T09:41:42.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-17T09:41:42.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-17T09:41:42.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-17T09:41:42.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-17T09:41:42.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-17T09:41:42.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-17T09:41:42.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-17T09:41:42.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.inplace_update_support: 0 2024-09-17T09:41:42.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-17T09:41:42.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-17T09:41:42.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-17T09:41:42.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-17T09:41:42.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.bloom_locality: 0 2024-09-17T09:41:42.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.max_successive_merges: 0 2024-09-17T09:41:42.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-17T09:41:42.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.paranoid_file_checks: 0 2024-09-17T09:41:42.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.force_consistency_checks: 1 2024-09-17T09:41:42.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.report_bg_io_stats: 0 2024-09-17T09:41:42.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.ttl: 2592000 2024-09-17T09:41:42.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-17T09:41:42.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-17T09:41:42.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-17T09:41:42.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.enable_blob_files: false 2024-09-17T09:41:42.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.min_blob_size: 0 2024-09-17T09:41:42.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.blob_file_size: 268435456 2024-09-17T09:41:42.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-17T09:41:42.621 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-17T09:41:42.621 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-17T09:41:42.621 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-17T09:41:42.621 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-17T09:41:42.621 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.blob_file_starting_level: 0 2024-09-17T09:41:42.621 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-17T09:41:42.621 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-09-17T09:41:42.621 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-17T09:41:42.622 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi078/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 11849, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-17T09:41:42.622 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-09-17T09:41:42.622 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 9a998869-c4bb-4bee-80b7-3026f81f1e89 2024-09-17T09:41:42.622 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726566102358232, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-09-17T09:41:42.622 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-09-17T09:41:42.622 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726566102372661, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 2666595, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11850, "largest_seqno": 12943, "table_properties": {"data_size": 2661094, "index_size": 3240, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1221, "raw_key_size": 10840, "raw_average_key_size": 23, "raw_value_size": 2650894, "raw_average_value_size": 5640, "num_data_blocks": 147, "num_entries": 470, "num_filter_entries": 470, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726566102, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "9a998869-c4bb-4bee-80b7-3026f81f1e89", "db_session_id": "2DO59TOJUXI6E6S6UCXJ", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-09-17T09:41:42.622 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726566102372868, "job": 1, "event": "recovery_finished"} 2024-09-17T09:41:42.622 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-09-17T09:41:42.622 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-17T09:41:42.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi078/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-17T09:41:42.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55f52fed4e00 2024-09-17T09:41:42.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: DB pointer 0x55f52ffd6000 2024-09-17T09:41:42.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-17T09:41:42.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-17T09:41:42.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** DB Stats ** 2024-09-17T09:41:42.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T09:41:42.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-17T09:41:42.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T09:41:42.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T09:41:42.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-17T09:41:42.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T09:41:42.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T09:41:42.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-17T09:41:42.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** Compaction Stats [default] ** 2024-09-17T09:41:42.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T09:41:42.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-17T09:41:42.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: L0 1/0 2.54 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 400.7 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-09-17T09:41:42.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: L6 1/0 7.07 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-17T09:41:42.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Sum 2/0 9.62 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 400.7 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-09-17T09:41:42.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 400.7 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-09-17T09:41:42.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-17T09:41:42.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** Compaction Stats [default] ** 2024-09-17T09:41:42.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T09:41:42.625 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T09:41:42.625 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 400.7 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-09-17T09:41:42.625 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-17T09:41:42.625 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-17T09:41:42.625 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-17T09:41:42.625 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T09:41:42.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-09-17T09:41:42.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T09:41:42.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T09:41:42.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T09:41:42.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-17T09:41:42.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative compaction: 0.00 GB write, 131.36 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T09:41:42.627 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval compaction: 0.00 GB write, 131.36 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T09:41:42.627 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T09:41:42.627 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Block cache BinnedLRUCache@0x55f52fea5610#2 capacity: 512.00 MB usage: 4.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2024-09-17T09:41:42.627 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.28 KB,0.000244379%) IndexBlock(1,3.33 KB,0.000634789%) Misc(1,0.00 KB,0%) 2024-09-17T09:41:42.627 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-17T09:41:42.627 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-17T09:41:42.628 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: starting mon.smithi078 rank 0 at public addrs [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] at bind addrs [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi078 fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 2024-09-17T09:41:42.628 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: mon.smithi078@-1(???) e2 preinit fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 2024-09-17T09:41:42.628 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: mon.smithi078@-1(???).mds e7 new map 2024-09-17T09:41:42.628 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: mon.smithi078@-1(???).mds e7 print_map 2024-09-17T09:41:42.628 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: e7 2024-09-17T09:41:42.628 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-17T09:41:42.628 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-17T09:41:42.629 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T09:41:42.629 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: legacy client fscid: 1 2024-09-17T09:41:42.629 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-17T09:41:42.629 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Filesystem 'foofs' (1) 2024-09-17T09:41:42.629 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: fs_name foofs 2024-09-17T09:41:42.629 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: epoch 5 2024-09-17T09:41:42.629 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-17T09:41:42.629 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: created 2024-09-17T09:36:28.738531+0000 2024-09-17T09:41:42.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: modified 2024-09-17T09:36:35.535121+0000 2024-09-17T09:41:42.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: tableserver 0 2024-09-17T09:41:42.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: root 0 2024-09-17T09:41:42.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: session_timeout 60 2024-09-17T09:41:42.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: session_autoclose 300 2024-09-17T09:41:42.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: max_file_size 1099511627776 2024-09-17T09:41:42.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: max_xattr_size 65536 2024-09-17T09:41:42.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: required_client_features {} 2024-09-17T09:41:42.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: last_failure 0 2024-09-17T09:41:42.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: last_failure_osd_epoch 0 2024-09-17T09:41:42.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T09:41:42.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: max_mds 1 2024-09-17T09:41:42.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: in 0 2024-09-17T09:41:42.632 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: up {0=14486} 2024-09-17T09:41:42.632 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: failed 2024-09-17T09:41:42.632 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: damaged 2024-09-17T09:41:42.632 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: stopped 2024-09-17T09:41:42.632 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: data_pools [3] 2024-09-17T09:41:42.632 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: metadata_pool 2 2024-09-17T09:41:42.633 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: inline_data disabled 2024-09-17T09:41:42.633 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: balancer 2024-09-17T09:41:42.633 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: bal_rank_mask -1 2024-09-17T09:41:42.633 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: standby_count_wanted 1 2024-09-17T09:41:42.633 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: qdb_cluster leader: 0 members: 2024-09-17T09:41:42.633 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: [mds.foofs.smithi078.jdrsup{0:14486} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.78:6834/2954767313,v1:172.21.15.78:6835/2954767313] compat {c=[1],r=[1],i=[7ff]}] 2024-09-17T09:41:42.633 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-17T09:41:42.633 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-17T09:41:42.633 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Standby daemons: 2024-09-17T09:41:42.634 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-17T09:41:42.634 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: [mds.foofs.smithi136.smslff{-1:24265} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.136:6832/1260769482,v1:172.21.15.136:6833/1260769482] compat {c=[1],r=[1],i=[7ff]}] 2024-09-17T09:41:42.634 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: mon.smithi078@-1(???).osd e72 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-17T09:41:42.634 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: mon.smithi078@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T09:41:42.634 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: mon.smithi078@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T09:41:42.634 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: mon.smithi078@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T09:41:42.634 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 ceph-mon[63787]: mon.smithi078@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-09-17T09:41:42.634 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 bash[63768]: 3ead101d3db1d111420cb146309cd04437cd607a750073cf14fdb72f34049737 2024-09-17T09:41:42.634 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:42 smithi078 systemd[1]: Started Ceph mon.smithi078 for 35c0cd22-74d7-11ef-bceb-c7b262605968. 2024-09-17T09:41:42.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 142888 11.87 MB/sec execute 157 sec latency 38.551 ms 2024-09-17T09:41:43.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 143630 11.90 MB/sec execute 158 sec latency 172.227 ms 2024-09-17T09:41:44.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 143950 11.86 MB/sec execute 159 sec latency 1172.284 ms 2024-09-17T09:41:45.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:45 smithi078 ceph-mon[63787]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-17T09:41:45.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:45 smithi078 ceph-mon[63787]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata", "id": "smithi136"}]: dispatch 2024-09-17T09:41:45.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:45 smithi078 ceph-mon[63787]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T09:41:45.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:45 smithi078 ceph-mon[63787]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T09:41:45.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:45 smithi078 ceph-mon[63787]: pgmap v28: 97 pgs: 97 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.4 MiB/s wr, 444 op/s 2024-09-17T09:41:45.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:45 smithi078 ceph-mon[63787]: pgmap v29: 97 pgs: 97 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 804 op/s 2024-09-17T09:41:45.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:45 smithi078 ceph-mon[63787]: pgmap v30: 97 pgs: 97 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 6.4 MiB/s wr, 589 op/s 2024-09-17T09:41:45.653 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:45 smithi078 ceph-mon[63787]: mon.smithi078 calling monitor election 2024-09-17T09:41:45.653 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:45 smithi078 ceph-mon[63787]: mon.smithi078 is new leader, mons smithi078,smithi136 in quorum (ranks 0,1) 2024-09-17T09:41:45.653 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:45 smithi078 ceph-mon[63787]: monmap epoch 2 2024-09-17T09:41:45.653 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:45 smithi078 ceph-mon[63787]: fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 2024-09-17T09:41:45.653 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:45 smithi078 ceph-mon[63787]: last_changed 2024-09-17T09:31:15.473949+0000 2024-09-17T09:41:45.653 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:45 smithi078 ceph-mon[63787]: created 2024-09-17T09:28:55.619057+0000 2024-09-17T09:41:45.653 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:45 smithi078 ceph-mon[63787]: min_mon_release 17 (quincy) 2024-09-17T09:41:45.653 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:45 smithi078 ceph-mon[63787]: election_strategy: 1 2024-09-17T09:41:45.653 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:45 smithi078 ceph-mon[63787]: 0: [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon.smithi078 2024-09-17T09:41:45.653 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:45 smithi078 ceph-mon[63787]: 1: [v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0] mon.smithi136 2024-09-17T09:41:45.653 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:45 smithi078 ceph-mon[63787]: fsmap foofs:1 {0=foofs.smithi078.jdrsup=up:active} 1 up:standby 2024-09-17T09:41:45.653 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:45 smithi078 ceph-mon[63787]: osdmap e72: 8 total, 8 up, 8 in 2024-09-17T09:41:45.654 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:45 smithi078 ceph-mon[63787]: mgrmap e35: smithi078.rsqofi(active, since 55s), standbys: smithi136.hicmlm 2024-09-17T09:41:45.654 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:45 smithi078 ceph-mon[63787]: overall HEALTH_OK 2024-09-17T09:41:45.654 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:45 smithi078 ceph-mon[63787]: from='mgr.14776 ' entity='' 2024-09-17T09:41:45.654 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:45 smithi078 ceph-mon[63787]: mgrmap e36: smithi078.rsqofi(active, since 55s), standbys: smithi136.hicmlm 2024-09-17T09:41:45.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:45 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-17T09:41:45.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:45 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "mon metadata", "id": "smithi136"}]: dispatch 2024-09-17T09:41:45.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:45 smithi136 ceph-mon[25953]: from='mgr.14776 172.21.15.78:0/196356747' entity='mgr.smithi078.rsqofi' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T09:41:45.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:45 smithi136 ceph-mon[25953]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T09:41:45.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:45 smithi136 ceph-mon[25953]: pgmap v28: 97 pgs: 97 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.4 MiB/s wr, 444 op/s 2024-09-17T09:41:45.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:45 smithi136 ceph-mon[25953]: pgmap v29: 97 pgs: 97 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 804 op/s 2024-09-17T09:41:45.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:45 smithi136 ceph-mon[25953]: pgmap v30: 97 pgs: 97 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 6.4 MiB/s wr, 589 op/s 2024-09-17T09:41:45.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:45 smithi136 ceph-mon[25953]: mon.smithi078 calling monitor election 2024-09-17T09:41:45.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:45 smithi136 ceph-mon[25953]: mon.smithi078 is new leader, mons smithi078,smithi136 in quorum (ranks 0,1) 2024-09-17T09:41:45.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:45 smithi136 ceph-mon[25953]: monmap epoch 2 2024-09-17T09:41:45.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:45 smithi136 ceph-mon[25953]: fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 2024-09-17T09:41:45.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:45 smithi136 ceph-mon[25953]: last_changed 2024-09-17T09:31:15.473949+0000 2024-09-17T09:41:45.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:45 smithi136 ceph-mon[25953]: created 2024-09-17T09:28:55.619057+0000 2024-09-17T09:41:45.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:45 smithi136 ceph-mon[25953]: min_mon_release 17 (quincy) 2024-09-17T09:41:45.728 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:45 smithi136 ceph-mon[25953]: election_strategy: 1 2024-09-17T09:41:45.728 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:45 smithi136 ceph-mon[25953]: 0: [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon.smithi078 2024-09-17T09:41:45.728 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:45 smithi136 ceph-mon[25953]: 1: [v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0] mon.smithi136 2024-09-17T09:41:45.728 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:45 smithi136 ceph-mon[25953]: fsmap foofs:1 {0=foofs.smithi078.jdrsup=up:active} 1 up:standby 2024-09-17T09:41:45.728 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:45 smithi136 ceph-mon[25953]: osdmap e72: 8 total, 8 up, 8 in 2024-09-17T09:41:45.728 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:45 smithi136 ceph-mon[25953]: mgrmap e35: smithi078.rsqofi(active, since 55s), standbys: smithi136.hicmlm 2024-09-17T09:41:45.728 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:45 smithi136 ceph-mon[25953]: overall HEALTH_OK 2024-09-17T09:41:45.728 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:45 smithi136 ceph-mon[25953]: from='mgr.14776 ' entity='' 2024-09-17T09:41:45.728 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:45 smithi136 ceph-mon[25953]: mgrmap e36: smithi078.rsqofi(active, since 55s), standbys: smithi136.hicmlm 2024-09-17T09:41:45.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 144149 11.81 MB/sec execute 160 sec latency 1782.053 ms 2024-09-17T09:41:46.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 144676 11.89 MB/sec execute 161 sec latency 33.033 ms 2024-09-17T09:41:47.431 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:47 smithi078 ceph-mon[63787]: mgrmap e37: smithi078.rsqofi(active, since 56s), standbys: smithi136.hicmlm 2024-09-17T09:41:47.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:47 smithi136 ceph-mon[25953]: mgrmap e37: smithi078.rsqofi(active, since 56s), standbys: smithi136.hicmlm 2024-09-17T09:41:47.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 145056 11.87 MB/sec execute 162 sec latency 112.260 ms 2024-09-17T09:41:48.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 145399 11.88 MB/sec execute 163 sec latency 72.812 ms 2024-09-17T09:41:49.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 145629 11.85 MB/sec execute 164 sec latency 63.142 ms 2024-09-17T09:41:50.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 145858 11.79 MB/sec execute 165 sec latency 51.328 ms 2024-09-17T09:41:51.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:51 smithi078 ceph-mon[63787]: Standby manager daemon smithi136.hicmlm restarted 2024-09-17T09:41:51.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:51 smithi078 ceph-mon[63787]: Standby manager daemon smithi136.hicmlm started 2024-09-17T09:41:51.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:51 smithi078 ceph-mon[63787]: from='mgr.? 172.21.15.136:0/916573956' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi136.hicmlm/crt"}]: dispatch 2024-09-17T09:41:51.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:51 smithi078 ceph-mon[63787]: from='mgr.? 172.21.15.136:0/916573956' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T09:41:51.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:51 smithi078 ceph-mon[63787]: from='mgr.? 172.21.15.136:0/916573956' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi136.hicmlm/key"}]: dispatch 2024-09-17T09:41:51.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:51 smithi078 ceph-mon[63787]: from='mgr.? 172.21.15.136:0/916573956' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T09:41:51.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:51 smithi136 ceph-mon[25953]: Standby manager daemon smithi136.hicmlm restarted 2024-09-17T09:41:51.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:51 smithi136 ceph-mon[25953]: Standby manager daemon smithi136.hicmlm started 2024-09-17T09:41:51.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:51 smithi136 ceph-mon[25953]: from='mgr.? 172.21.15.136:0/916573956' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi136.hicmlm/crt"}]: dispatch 2024-09-17T09:41:51.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:51 smithi136 ceph-mon[25953]: from='mgr.? 172.21.15.136:0/916573956' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T09:41:51.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:51 smithi136 ceph-mon[25953]: from='mgr.? 172.21.15.136:0/916573956' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi136.hicmlm/key"}]: dispatch 2024-09-17T09:41:51.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:51 smithi136 ceph-mon[25953]: from='mgr.? 172.21.15.136:0/916573956' entity='mgr.smithi136.hicmlm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T09:41:51.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 146172 11.77 MB/sec execute 166 sec latency 66.308 ms 2024-09-17T09:41:52.675 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:52 smithi078 ceph-mon[63787]: mgrmap e38: smithi078.rsqofi(active, since 61s), standbys: smithi136.hicmlm 2024-09-17T09:41:52.675 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:52 smithi078 ceph-mon[63787]: Active manager daemon smithi078.rsqofi restarted 2024-09-17T09:41:52.675 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:52 smithi078 ceph-mon[63787]: Activating manager daemon smithi078.rsqofi 2024-09-17T09:41:52.675 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:52 smithi078 ceph-mon[63787]: osdmap e73: 8 total, 8 up, 8 in 2024-09-17T09:41:52.675 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:52 smithi078 ceph-mon[63787]: mgrmap e39: smithi078.rsqofi(active, starting, since 0.00852102s), standbys: smithi136.hicmlm 2024-09-17T09:41:52.675 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mon metadata", "id": "smithi078"} : dispatch 2024-09-17T09:41:52.675 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mon metadata", "id": "smithi136"} : dispatch 2024-09-17T09:41:52.676 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mds metadata", "who": "foofs.smithi078.jdrsup"} : dispatch 2024-09-17T09:41:52.676 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mds metadata", "who": "foofs.smithi136.smslff"} : dispatch 2024-09-17T09:41:52.676 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mgr metadata", "who": "smithi078.rsqofi", "id": "smithi078.rsqofi"} : dispatch 2024-09-17T09:41:52.676 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mgr metadata", "who": "smithi136.hicmlm", "id": "smithi136.hicmlm"} : dispatch 2024-09-17T09:41:52.676 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-17T09:41:52.676 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-17T09:41:52.676 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-17T09:41:52.676 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-17T09:41:52.676 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-17T09:41:52.676 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-17T09:41:52.676 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-17T09:41:52.676 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-17T09:41:52.676 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mds metadata"} : dispatch 2024-09-17T09:41:52.676 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata"} : dispatch 2024-09-17T09:41:52.677 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mon metadata"} : dispatch 2024-09-17T09:41:52.677 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:52 smithi078 ceph-mon[63787]: Manager daemon smithi078.rsqofi is now available 2024-09-17T09:41:52.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 146404 11.74 MB/sec execute 167 sec latency 64.211 ms 2024-09-17T09:41:52.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:52 smithi136 ceph-mon[25953]: mgrmap e38: smithi078.rsqofi(active, since 61s), standbys: smithi136.hicmlm 2024-09-17T09:41:52.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:52 smithi136 ceph-mon[25953]: Active manager daemon smithi078.rsqofi restarted 2024-09-17T09:41:52.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:52 smithi136 ceph-mon[25953]: Activating manager daemon smithi078.rsqofi 2024-09-17T09:41:52.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:52 smithi136 ceph-mon[25953]: osdmap e73: 8 total, 8 up, 8 in 2024-09-17T09:41:52.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:52 smithi136 ceph-mon[25953]: mgrmap e39: smithi078.rsqofi(active, starting, since 0.00852102s), standbys: smithi136.hicmlm 2024-09-17T09:41:52.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:52 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mon metadata", "id": "smithi078"} : dispatch 2024-09-17T09:41:52.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:52 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mon metadata", "id": "smithi136"} : dispatch 2024-09-17T09:41:52.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:52 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mds metadata", "who": "foofs.smithi078.jdrsup"} : dispatch 2024-09-17T09:41:52.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:52 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mds metadata", "who": "foofs.smithi136.smslff"} : dispatch 2024-09-17T09:41:52.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:52 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mgr metadata", "who": "smithi078.rsqofi", "id": "smithi078.rsqofi"} : dispatch 2024-09-17T09:41:52.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:52 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mgr metadata", "who": "smithi136.hicmlm", "id": "smithi136.hicmlm"} : dispatch 2024-09-17T09:41:52.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:52 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-17T09:41:52.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:52 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-17T09:41:52.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:52 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-17T09:41:52.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:52 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-17T09:41:52.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:52 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-17T09:41:52.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:52 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-17T09:41:52.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:52 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-17T09:41:52.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:52 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-17T09:41:52.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:52 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mds metadata"} : dispatch 2024-09-17T09:41:52.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:52 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata"} : dispatch 2024-09-17T09:41:52.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:52 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mon metadata"} : dispatch 2024-09-17T09:41:52.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:52 smithi136 ceph-mon[25953]: Manager daemon smithi078.rsqofi is now available 2024-09-17T09:41:53.708 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:53 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:41:53.709 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:53 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:41:53.709 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:53 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-17T09:41:53.709 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:53 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/mirror_snapshot_schedule"} : dispatch 2024-09-17T09:41:53.709 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:53 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/trash_purge_schedule"} : dispatch 2024-09-17T09:41:53.709 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:53 smithi078 ceph-mon[63787]: mgrmap e40: smithi078.rsqofi(active, since 1.03499s), standbys: smithi136.hicmlm 2024-09-17T09:41:53.709 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:53 smithi078 ceph-mon[63787]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T09:41:53.709 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:53 smithi078 ceph-mon[63787]: pgmap v3: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-17T09:41:53.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 146607 11.69 MB/sec execute 168 sec latency 60.067 ms 2024-09-17T09:41:53.974 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:53 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:41:53.974 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:53 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:41:53.974 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:53 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-17T09:41:53.974 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:53 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/mirror_snapshot_schedule"} : dispatch 2024-09-17T09:41:53.974 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:53 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rsqofi/trash_purge_schedule"} : dispatch 2024-09-17T09:41:53.974 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:53 smithi136 ceph-mon[25953]: mgrmap e40: smithi078.rsqofi(active, since 1.03499s), standbys: smithi136.hicmlm 2024-09-17T09:41:53.974 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:53 smithi136 ceph-mon[25953]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T09:41:53.974 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:53 smithi136 ceph-mon[25953]: pgmap v3: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-17T09:41:54.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 146922 11.68 MB/sec execute 169 sec latency 61.911 ms 2024-09-17T09:41:55.372 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:55 smithi136 ceph-mon[25953]: [17/Sep/2024:09:41:54] ENGINE Bus STARTING 2024-09-17T09:41:55.373 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:55 smithi136 ceph-mon[25953]: pgmap v4: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-17T09:41:55.373 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:55 smithi136 ceph-mon[25953]: [17/Sep/2024:09:41:54] ENGINE Serving on https://172.21.15.78:7150 2024-09-17T09:41:55.492 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:55 smithi078 ceph-mon[63787]: [17/Sep/2024:09:41:54] ENGINE Bus STARTING 2024-09-17T09:41:55.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:55 smithi078 ceph-mon[63787]: pgmap v4: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-17T09:41:55.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:55 smithi078 ceph-mon[63787]: [17/Sep/2024:09:41:54] ENGINE Serving on https://172.21.15.78:7150 2024-09-17T09:41:55.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:55 smithi078 ceph-mon[63787]: [17/Sep/2024:09:41:54] ENGINE Error in HTTPServer.serve 2024-09-17T09:41:55.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Traceback (most recent call last): 2024-09-17T09:41:55.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T09:41:55.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self._connections.run(self.expiration_interval) 2024-09-17T09:41:55.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T09:41:55.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self._run(expiration_interval) 2024-09-17T09:41:55.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T09:41:55.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T09:41:55.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T09:41:55.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T09:41:55.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T09:41:55.494 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: s = self.context.wrap_socket( 2024-09-17T09:41:55.494 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T09:41:55.494 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: return self.sslsocket_class._create( 2024-09-17T09:41:55.494 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T09:41:55.494 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self.do_handshake() 2024-09-17T09:41:55.494 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T09:41:55.494 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self._sslobj.do_handshake() 2024-09-17T09:41:55.494 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T09:41:55.494 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:55 smithi078 ceph-mon[63787]: mgrmap e41: smithi078.rsqofi(active, since 2s), standbys: smithi136.hicmlm 2024-09-17T09:41:55.627 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:55 smithi136 ceph-mon[25953]: [17/Sep/2024:09:41:54] ENGINE Error in HTTPServer.serve 2024-09-17T09:41:55.627 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: Traceback (most recent call last): 2024-09-17T09:41:55.627 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T09:41:55.627 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: self._connections.run(self.expiration_interval) 2024-09-17T09:41:55.627 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T09:41:55.627 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: self._run(expiration_interval) 2024-09-17T09:41:55.627 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T09:41:55.627 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T09:41:55.627 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T09:41:55.627 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T09:41:55.627 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T09:41:55.628 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: s = self.context.wrap_socket( 2024-09-17T09:41:55.628 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T09:41:55.628 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: return self.sslsocket_class._create( 2024-09-17T09:41:55.628 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T09:41:55.628 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: self.do_handshake() 2024-09-17T09:41:55.628 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T09:41:55.628 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: self._sslobj.do_handshake() 2024-09-17T09:41:55.628 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T09:41:55.628 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:55 smithi136 ceph-mon[25953]: mgrmap e41: smithi078.rsqofi(active, since 2s), standbys: smithi136.hicmlm 2024-09-17T09:41:55.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 147241 11.62 MB/sec execute 170 sec latency 62.896 ms 2024-09-17T09:41:56.536 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:56 smithi078 ceph-mon[63787]: [17/Sep/2024:09:41:54] ENGINE Serving on http://172.21.15.78:8765 2024-09-17T09:41:56.536 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:56 smithi078 ceph-mon[63787]: [17/Sep/2024:09:41:54] ENGINE Bus STARTED 2024-09-17T09:41:56.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:56 smithi136 ceph-mon[25953]: [17/Sep/2024:09:41:54] ENGINE Serving on http://172.21.15.78:8765 2024-09-17T09:41:56.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:56 smithi136 ceph-mon[25953]: [17/Sep/2024:09:41:54] ENGINE Bus STARTED 2024-09-17T09:41:56.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 147544 11.58 MB/sec execute 171 sec latency 35.677 ms 2024-09-17T09:41:57.468 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:57 smithi078 ceph-mon[63787]: pgmap v5: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-17T09:41:57.468 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:57 smithi078 ceph-mon[63787]: mgrmap e42: smithi078.rsqofi(active, since 4s), standbys: smithi136.hicmlm 2024-09-17T09:41:57.468 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:57 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:57.468 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:57 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:57.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:57 smithi136 ceph-mon[25953]: pgmap v5: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-17T09:41:57.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:57 smithi136 ceph-mon[25953]: mgrmap e42: smithi078.rsqofi(active, since 4s), standbys: smithi136.hicmlm 2024-09-17T09:41:57.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:57 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:57.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:57 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:57.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 147721 11.56 MB/sec execute 172 sec latency 92.579 ms 2024-09-17T09:41:58.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 148186 11.60 MB/sec execute 173 sec latency 67.275 ms 2024-09-17T09:41:59.104 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:58 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:59.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:58 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:59.105 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:41:58 smithi078 ceph-mon[63787]: pgmap v6: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-17T09:41:59.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:58 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:59.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:58 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:41:59.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:41:58 smithi136 ceph-mon[25953]: pgmap v6: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-17T09:41:59.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 148625 11.60 MB/sec execute 174 sec latency 58.849 ms 2024-09-17T09:42:00.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 148952 11.61 MB/sec execute 175 sec latency 39.134 ms 2024-09-17T09:42:01.627 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:01 smithi078 ceph-mon[63787]: pgmap v7: 97 pgs: 97 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 29 KiB/s rd, 2.5 MiB/s wr, 305 op/s 2024-09-17T09:42:01.628 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:01 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:01.628 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:01 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:01.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:01 smithi136 ceph-mon[25953]: pgmap v7: 97 pgs: 97 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 29 KiB/s rd, 2.5 MiB/s wr, 305 op/s 2024-09-17T09:42:01.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:01 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:01.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:01 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:01.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 149220 11.59 MB/sec execute 176 sec latency 93.065 ms 2024-09-17T09:42:02.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 149518 11.55 MB/sec execute 177 sec latency 73.285 ms 2024-09-17T09:42:03.168 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:03 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:03.168 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:03 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:03.168 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:03 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:03.168 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:03 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:03.168 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:03 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:03.168 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:03 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:42:03.168 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:03 smithi078 ceph-mon[63787]: Updating smithi078:/etc/ceph/ceph.conf 2024-09-17T09:42:03.168 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:03 smithi078 ceph-mon[63787]: Updating smithi136:/etc/ceph/ceph.conf 2024-09-17T09:42:03.168 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:03 smithi078 ceph-mon[63787]: pgmap v8: 97 pgs: 97 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 2.0 MiB/s wr, 236 op/s 2024-09-17T09:42:03.169 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:03 smithi078 ceph-mon[63787]: Updating smithi136:/var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T09:42:03.169 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:03 smithi078 ceph-mon[63787]: Updating smithi078:/var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T09:42:03.169 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:03 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:03.169 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:03 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:03.169 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:03 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:03.169 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:03 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:03.169 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:03 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:03.169 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:03 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:03.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:03 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:03.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:03 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:03.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:03 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:03.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:03 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:03.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:03 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:03.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:03 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:42:03.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:03 smithi136 ceph-mon[25953]: Updating smithi078:/etc/ceph/ceph.conf 2024-09-17T09:42:03.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:03 smithi136 ceph-mon[25953]: Updating smithi136:/etc/ceph/ceph.conf 2024-09-17T09:42:03.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:03 smithi136 ceph-mon[25953]: pgmap v8: 97 pgs: 97 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 2.0 MiB/s wr, 236 op/s 2024-09-17T09:42:03.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:03 smithi136 ceph-mon[25953]: Updating smithi136:/var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T09:42:03.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:03 smithi136 ceph-mon[25953]: Updating smithi078:/var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T09:42:03.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:03 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:03.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:03 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:03.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:03 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:03.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:03 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:03.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:03 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:03.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:03 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:03.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 149862 11.54 MB/sec execute 178 sec latency 48.056 ms 2024-09-17T09:42:04.175 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-17T09:42:04.442 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:04 smithi078 ceph-mon[63787]: Updating smithi136:/etc/ceph/ceph.client.admin.keyring 2024-09-17T09:42:04.443 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:04 smithi078 ceph-mon[63787]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-09-17T09:42:04.443 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:04 smithi078 ceph-mon[63787]: Updating smithi136:/var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T09:42:04.443 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:04 smithi078 ceph-mon[63787]: Updating smithi078:/var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T09:42:04.443 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:04 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:42:04.443 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:04 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:42:04.443 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:04 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "quorum_status"} : dispatch 2024-09-17T09:42:04.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:04 smithi136 ceph-mon[25953]: Updating smithi136:/etc/ceph/ceph.client.admin.keyring 2024-09-17T09:42:04.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:04 smithi136 ceph-mon[25953]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-09-17T09:42:04.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:04 smithi136 ceph-mon[25953]: Updating smithi136:/var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T09:42:04.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:04 smithi136 ceph-mon[25953]: Updating smithi078:/var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T09:42:04.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:04 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:42:04.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:04 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:42:04.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:04 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "quorum_status"} : dispatch 2024-09-17T09:42:04.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 150062 11.50 MB/sec execute 179 sec latency 50.225 ms 2024-09-17T09:42:05.113 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:42:05.114 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (2m) 3s ago 11m 19.2M - 0.27.0 11f11916f8cd 97db2373ce92 2024-09-17T09:42:05.114 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (11m) 3s ago 11m 6400k - 17.2.0 e1d6a67b021e 46d44333624b 2024-09-17T09:42:05.114 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi136 smithi136 running (10m) 7s ago 10m 6395k - 17.2.0 e1d6a67b021e 56ce035cb482 2024-09-17T09:42:05.114 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (2m) 3s ago 11m 53.1M - dad864ee21e9 db74b0f8e0a7 2024-09-17T09:42:05.114 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.jdrsup smithi078 running (5m) 3s ago 5m 37.5M - 17.2.0 e1d6a67b021e 8f834e15ba88 2024-09-17T09:42:05.114 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi136.smslff smithi136 running (5m) 7s ago 5m 15.5M - 17.2.0 e1d6a67b021e 0c20e2b328d3 2024-09-17T09:42:05.114 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rsqofi smithi078 *:8443,9283,8765 running (95s) 3s ago 12m 488M - 19.3.0-4963-ge646f729 f271290a0950 7dda3108c9a0 2024-09-17T09:42:05.114 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi136.hicmlm smithi136 *:8443,9283,8765 running (42s) 7s ago 10m 452M - 19.3.0-4963-ge646f729 f271290a0950 fc518b58e07a 2024-09-17T09:42:05.114 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (23s) 3s ago 13m 32.4M 2048M 19.3.0-4963-ge646f729 f271290a0950 3ead101d3db1 2024-09-17T09:42:05.114 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi136 smithi136 running (10m) 7s ago 10m 52.3M 2048M 17.2.0 e1d6a67b021e 1bfdfb0617bb 2024-09-17T09:42:05.114 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.tcwbvi smithi078 *:2049 running (5m) 3s ago 5m 264M - 4.0 e1d6a67b021e 30c87cb46555 2024-09-17T09:42:05.115 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi136.regjvc smithi136 *:2049 running (5m) 7s ago 5m 87.5M - 4.0 e1d6a67b021e b88e4dc24376 2024-09-17T09:42:05.115 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (2m) 3s ago 11m 11.4M - 1.7.0 72c9c2088986 52dad3adeeaa 2024-09-17T09:42:05.115 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi136 smithi136 *:9100 running (105s) 7s ago 10m 11.4M - 1.7.0 72c9c2088986 f2bc5856e35a 2024-09-17T09:42:05.115 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (10m) 3s ago 10m 498M 1936M 17.2.0 e1d6a67b021e b4ca42f8477d 2024-09-17T09:42:05.115 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (9m) 3s ago 9m 451M 1936M 17.2.0 e1d6a67b021e 351873014de4 2024-09-17T09:42:05.115 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (9m) 3s ago 9m 311M 1936M 17.2.0 e1d6a67b021e 07eb71e4fa57 2024-09-17T09:42:05.115 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (8m) 3s ago 8m 469M 1936M 17.2.0 e1d6a67b021e 1b3a8be89881 2024-09-17T09:42:05.115 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi136 running (8m) 7s ago 8m 504M 2704M 17.2.0 e1d6a67b021e a0c5b4de2414 2024-09-17T09:42:05.115 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi136 running (7m) 7s ago 7m 561M 2704M 17.2.0 e1d6a67b021e 30a3049955f9 2024-09-17T09:42:05.115 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi136 running (7m) 7s ago 7m 480M 2704M 17.2.0 e1d6a67b021e 1698e1567c33 2024-09-17T09:42:05.115 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi136 running (7m) 7s ago 7m 370M 2704M 17.2.0 e1d6a67b021e 5d3b08cecd78 2024-09-17T09:42:05.115 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (48s) 3s ago 10m 46.3M - 2.51.0 1d3b7f56885b 6e5e8c721eb0 2024-09-17T09:42:05.464 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:05 smithi136 ceph-mon[25953]: from='client.34130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:42:05.465 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:05 smithi136 ceph-mon[25953]: pgmap v9: 97 pgs: 97 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 21 KiB/s rd, 4.2 MiB/s wr, 416 op/s 2024-09-17T09:42:05.465 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:05 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:05.465 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:05 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-17T09:42:05.465 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:05 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-17T09:42:05.465 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:05 smithi136 ceph-mon[25953]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:05.465 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:05 smithi078 ceph-mon[63787]: from='client.34130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:42:05.465 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:05 smithi078 ceph-mon[63787]: pgmap v9: 97 pgs: 97 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 21 KiB/s rd, 4.2 MiB/s wr, 416 op/s 2024-09-17T09:42:05.466 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:05 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:05.466 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:05 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-17T09:42:05.466 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:05 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-17T09:42:05.466 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:05 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:05.682 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:42:05.682 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-17T09:42:05.682 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T09:42:05.682 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 1 2024-09-17T09:42:05.682 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:42:05.682 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-17T09:42:05.682 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:42:05.682 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:42:05.682 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-17T09:42:05.682 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T09:42:05.683 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:42:05.683 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-17T09:42:05.683 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:42:05.683 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:42:05.683 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-17T09:42:05.683 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-09-17T09:42:05.683 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 3 2024-09-17T09:42:05.683 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-17T09:42:05.683 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:42:05.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 150359 11.48 MB/sec execute 180 sec latency 54.543 ms 2024-09-17T09:42:06.185 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:42:06.185 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407", 2024-09-17T09:42:06.185 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-17T09:42:06.185 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T09:42:06.186 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-17T09:42:06.186 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr" 2024-09-17T09:42:06.186 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-17T09:42:06.186 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "3/23 daemons upgraded", 2024-09-17T09:42:06.186 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading mon daemons", 2024-09-17T09:42:06.186 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-17T09:42:06.186 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:42:06.484 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:06 smithi078 ceph-mon[63787]: from='client.24611 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:42:06.484 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:06 smithi078 ceph-mon[63787]: Upgrade: Updating mon.smithi136 2024-09-17T09:42:06.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:06 smithi078 ceph-mon[63787]: Deploying daemon mon.smithi136 on smithi136 2024-09-17T09:42:06.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:06 smithi078 ceph-mon[63787]: from='client.24615 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:42:06.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:06 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/1446421666' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:42:06.649 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:06 smithi136 ceph-mon[25953]: from='client.24611 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:42:06.649 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:06 smithi136 ceph-mon[25953]: Upgrade: Updating mon.smithi136 2024-09-17T09:42:06.649 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:06 smithi136 ceph-mon[25953]: Deploying daemon mon.smithi136 on smithi136 2024-09-17T09:42:06.649 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:06 smithi136 ceph-mon[25953]: from='client.24615 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:42:06.649 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:06 smithi136 ceph-mon[25953]: from='client.? 172.21.15.78:0/1446421666' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:42:06.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 150685 11.44 MB/sec execute 181 sec latency 50.627 ms 2024-09-17T09:42:06.800 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-17T09:42:07.193 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:06 smithi136 systemd[1]: Stopping Ceph mon.smithi136 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:42:07.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:07 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi136[25949]: 2024-09-17T09:42:07.191+0000 7f5ad8cc7700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi136 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:42:07.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:07 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi136[25949]: 2024-09-17T09:42:07.191+0000 7f5ad8cc7700 -1 mon.smithi136@1(peon) e2 *** Got Signal Terminated *** 2024-09-17T09:42:07.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:07 smithi136 podman[49410]: 2024-09-17 09:42:07.232470299 +0000 UTC m=+0.158981207 container died 1bfdfb0617bb75c95b5e1274f36506e0ba322d28d461a79dad5fae41725f666d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi136, GIT_BRANCH=HEAD, io.openshift.expose-services=, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, name=centos-stream, distribution-scope=public, RELEASE=HEAD, com.redhat.component=centos-stream-container, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., release=754, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-17T09:42:07.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 151014 11.40 MB/sec execute 182 sec latency 50.121 ms 2024-09-17T09:42:08.121 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:07 smithi136 podman[49410]: 2024-09-17 09:42:07.859519993 +0000 UTC m=+0.786030909 container cleanup 1bfdfb0617bb75c95b5e1274f36506e0ba322d28d461a79dad5fae41725f666d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi136, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, vendor=Red Hat, Inc., GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, ceph=True, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-17T09:42:08.121 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:07 smithi136 bash[49410]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi136 2024-09-17T09:42:08.376 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:08 smithi136 podman[49422]: 2024-09-17 09:42:08.120030564 +0000 UTC m=+0.878274674 container remove 1bfdfb0617bb75c95b5e1274f36506e0ba322d28d461a79dad5fae41725f666d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi136, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , io.openshift.expose-services=, GIT_CLEAN=True, version=8, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0) 2024-09-17T09:42:08.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:08 smithi136 systemd[1]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@mon.smithi136.service: Deactivated successfully. 2024-09-17T09:42:08.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:08 smithi136 systemd[1]: Stopped Ceph mon.smithi136 for 35c0cd22-74d7-11ef-bceb-c7b262605968. 2024-09-17T09:42:08.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:08 smithi136 systemd[1]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@mon.smithi136.service: Consumed 7.045s CPU time. 2024-09-17T09:42:08.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:08 smithi136 systemd[1]: Starting Ceph mon.smithi136 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:42:08.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 151267 11.39 MB/sec execute 183 sec latency 49.865 ms 2024-09-17T09:42:09.151 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:08 smithi136 podman[49542]: 2024-09-17 09:42:08.749819013 +0000 UTC m=+0.018732096 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:42:09.151 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:08 smithi136 podman[49542]: 2024-09-17 09:42:08.880091816 +0000 UTC m=+0.149004880 container create 4fce88842b79c82363aecf01ca6b4c6264d292d49e481ad99acacb35a9646d5a (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi136, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-17T09:42:09.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 podman[49542]: 2024-09-17 09:42:09.149898543 +0000 UTC m=+0.418811616 container init 4fce88842b79c82363aecf01ca6b4c6264d292d49e481ad99acacb35a9646d5a (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi136, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-17T09:42:09.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 podman[49542]: 2024-09-17 09:42:09.154387299 +0000 UTC m=+0.423300364 container start 4fce88842b79c82363aecf01ca6b4c6264d292d49e481ad99acacb35a9646d5a (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi136, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72) 2024-09-17T09:42:09.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: set uid:gid to 167:167 (ceph:ceph) 2024-09-17T09:42:09.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev), process ceph-mon, pid 2 2024-09-17T09:42:09.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: pidfile_write: ignore empty --pid-file 2024-09-17T09:42:09.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: load: jerasure load: lrc 2024-09-17T09:42:09.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: RocksDB version: 7.9.2 2024-09-17T09:42:09.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Git sha 0 2024-09-17T09:42:09.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Compile date 2024-09-16 20:37:15 2024-09-17T09:42:09.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: DB SUMMARY 2024-09-17T09:42:09.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: DB Session ID: 1GMHPJHYL654PBB1FZPW 2024-09-17T09:42:09.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: CURRENT file: CURRENT 2024-09-17T09:42:09.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: IDENTITY file: IDENTITY 2024-09-17T09:42:09.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: MANIFEST file: MANIFEST-000009 size: 904 Bytes 2024-09-17T09:42:09.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi136/store.db dir, Total Num: 1, files: 000024.sst 2024-09-17T09:42:09.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi136/store.db: 000022.log size: 7743014 ; 2024-09-17T09:42:09.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.error_if_exists: 0 2024-09-17T09:42:09.478 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.create_if_missing: 0 2024-09-17T09:42:09.478 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.paranoid_checks: 1 2024-09-17T09:42:09.478 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-17T09:42:09.478 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-17T09:42:09.478 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-17T09:42:09.478 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.env: 0x55d700554120 2024-09-17T09:42:09.478 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.fs: PosixFileSystem 2024-09-17T09:42:09.478 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.info_log: 0x55d702cfd8e0 2024-09-17T09:42:09.478 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.max_file_opening_threads: 16 2024-09-17T09:42:09.479 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.statistics: (nil) 2024-09-17T09:42:09.480 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.use_fsync: 0 2024-09-17T09:42:09.480 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.max_log_file_size: 0 2024-09-17T09:42:09.480 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-17T09:42:09.480 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-17T09:42:09.480 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.keep_log_file_num: 1000 2024-09-17T09:42:09.480 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.recycle_log_file_num: 0 2024-09-17T09:42:09.480 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.allow_fallocate: 1 2024-09-17T09:42:09.480 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.allow_mmap_reads: 0 2024-09-17T09:42:09.480 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.allow_mmap_writes: 0 2024-09-17T09:42:09.480 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.use_direct_reads: 0 2024-09-17T09:42:09.480 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-17T09:42:09.481 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.create_missing_column_families: 0 2024-09-17T09:42:09.481 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.db_log_dir: 2024-09-17T09:42:09.481 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.wal_dir: 2024-09-17T09:42:09.481 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-17T09:42:09.481 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-17T09:42:09.481 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-17T09:42:09.481 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-17T09:42:09.481 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-17T09:42:09.481 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-17T09:42:09.481 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.advise_random_on_open: 1 2024-09-17T09:42:09.481 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.db_write_buffer_size: 0 2024-09-17T09:42:09.481 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.write_buffer_manager: 0x55d702d01b80 2024-09-17T09:42:09.481 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-17T09:42:09.482 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-17T09:42:09.482 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-17T09:42:09.482 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.rate_limiter: (nil) 2024-09-17T09:42:09.482 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-17T09:42:09.482 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.wal_recovery_mode: 2 2024-09-17T09:42:09.482 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.enable_thread_tracking: 0 2024-09-17T09:42:09.482 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.enable_pipelined_write: 0 2024-09-17T09:42:09.482 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.unordered_write: 0 2024-09-17T09:42:09.482 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-17T09:42:09.482 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-17T09:42:09.482 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-17T09:42:09.482 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-17T09:42:09.482 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.row_cache: None 2024-09-17T09:42:09.482 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.wal_filter: None 2024-09-17T09:42:09.483 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-17T09:42:09.483 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.allow_ingest_behind: 0 2024-09-17T09:42:09.483 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.two_write_queues: 0 2024-09-17T09:42:09.483 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.manual_wal_flush: 0 2024-09-17T09:42:09.483 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.wal_compression: 0 2024-09-17T09:42:09.483 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.atomic_flush: 0 2024-09-17T09:42:09.483 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-17T09:42:09.483 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-17T09:42:09.483 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-17T09:42:09.483 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.log_readahead_size: 0 2024-09-17T09:42:09.483 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-17T09:42:09.483 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.best_efforts_recovery: 0 2024-09-17T09:42:09.483 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-17T09:42:09.484 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-17T09:42:09.484 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.allow_data_in_errors: 0 2024-09-17T09:42:09.484 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.db_host_id: __hostname__ 2024-09-17T09:42:09.484 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-17T09:42:09.484 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.max_background_jobs: 2 2024-09-17T09:42:09.484 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.max_background_compactions: -1 2024-09-17T09:42:09.484 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.max_subcompactions: 1 2024-09-17T09:42:09.484 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-17T09:42:09.484 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-17T09:42:09.484 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-17T09:42:09.484 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.max_total_wal_size: 0 2024-09-17T09:42:09.484 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-17T09:42:09.484 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-17T09:42:09.485 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-17T09:42:09.485 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-17T09:42:09.485 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.max_open_files: -1 2024-09-17T09:42:09.485 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.bytes_per_sync: 0 2024-09-17T09:42:09.485 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-17T09:42:09.485 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-17T09:42:09.485 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.compaction_readahead_size: 0 2024-09-17T09:42:09.485 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.max_background_flushes: -1 2024-09-17T09:42:09.485 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Compression algorithms supported: 2024-09-17T09:42:09.485 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: kZSTD supported: 0 2024-09-17T09:42:09.485 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: kXpressCompression supported: 0 2024-09-17T09:42:09.485 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: kBZip2Compression supported: 0 2024-09-17T09:42:09.485 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-17T09:42:09.486 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: kLZ4Compression supported: 1 2024-09-17T09:42:09.486 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: kZlibCompression supported: 1 2024-09-17T09:42:09.486 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: kLZ4HCCompression supported: 1 2024-09-17T09:42:09.486 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: kSnappyCompression supported: 1 2024-09-17T09:42:09.486 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-17T09:42:09.486 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-17T09:42:09.486 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi136/store.db/MANIFEST-000009 2024-09-17T09:42:09.486 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-17T09:42:09.486 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-17T09:42:09.486 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.merge_operator: 2024-09-17T09:42:09.486 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.compaction_filter: None 2024-09-17T09:42:09.486 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.compaction_filter_factory: None 2024-09-17T09:42:09.486 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.sst_partitioner_factory: None 2024-09-17T09:42:09.487 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-17T09:42:09.487 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-17T09:42:09.487 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55d702cfd540) 2024-09-17T09:42:09.487 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: cache_index_and_filter_blocks: 1 2024-09-17T09:42:09.487 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-17T09:42:09.487 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-17T09:42:09.487 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: pin_top_level_index_and_filter: 1 2024-09-17T09:42:09.487 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: index_type: 0 2024-09-17T09:42:09.487 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: data_block_index_type: 0 2024-09-17T09:42:09.487 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: index_shortening: 1 2024-09-17T09:42:09.487 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-17T09:42:09.487 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: checksum: 4 2024-09-17T09:42:09.487 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: no_block_cache: 0 2024-09-17T09:42:09.487 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: block_cache: 0x55d702cf1610 2024-09-17T09:42:09.488 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: block_cache_name: BinnedLRUCache 2024-09-17T09:42:09.488 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: block_cache_options: 2024-09-17T09:42:09.488 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: capacity : 536870912 2024-09-17T09:42:09.488 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: num_shard_bits : 4 2024-09-17T09:42:09.488 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: strict_capacity_limit : 0 2024-09-17T09:42:09.488 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: high_pri_pool_ratio: 0.000 2024-09-17T09:42:09.488 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: block_cache_compressed: (nil) 2024-09-17T09:42:09.488 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: persistent_cache: (nil) 2024-09-17T09:42:09.488 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: block_size: 4096 2024-09-17T09:42:09.488 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: block_size_deviation: 10 2024-09-17T09:42:09.488 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: block_restart_interval: 16 2024-09-17T09:42:09.488 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: index_block_restart_interval: 1 2024-09-17T09:42:09.488 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: metadata_block_size: 4096 2024-09-17T09:42:09.489 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: partition_filters: 0 2024-09-17T09:42:09.489 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: use_delta_encoding: 1 2024-09-17T09:42:09.489 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: filter_policy: bloomfilter 2024-09-17T09:42:09.489 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: whole_key_filtering: 1 2024-09-17T09:42:09.489 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: verify_compression: 0 2024-09-17T09:42:09.489 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: read_amp_bytes_per_bit: 0 2024-09-17T09:42:09.490 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: format_version: 5 2024-09-17T09:42:09.490 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: enable_index_compression: 1 2024-09-17T09:42:09.490 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: block_align: 0 2024-09-17T09:42:09.490 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: max_auto_readahead_size: 262144 2024-09-17T09:42:09.490 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: prepopulate_block_cache: 0 2024-09-17T09:42:09.490 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: initial_auto_readahead_size: 8192 2024-09-17T09:42:09.490 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: num_file_reads_for_auto_readahead: 2 2024-09-17T09:42:09.490 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.write_buffer_size: 33554432 2024-09-17T09:42:09.490 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.max_write_buffer_number: 2 2024-09-17T09:42:09.490 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.compression: NoCompression 2024-09-17T09:42:09.490 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.bottommost_compression: Disabled 2024-09-17T09:42:09.491 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.prefix_extractor: nullptr 2024-09-17T09:42:09.491 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-17T09:42:09.491 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.num_levels: 7 2024-09-17T09:42:09.491 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-17T09:42:09.491 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-17T09:42:09.491 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-17T09:42:09.491 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-17T09:42:09.491 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-17T09:42:09.491 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-17T09:42:09.491 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-17T09:42:09.491 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-17T09:42:09.491 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-17T09:42:09.491 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-17T09:42:09.491 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-17T09:42:09.492 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-17T09:42:09.492 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-17T09:42:09.492 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.compression_opts.level: 32767 2024-09-17T09:42:09.492 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.compression_opts.strategy: 0 2024-09-17T09:42:09.492 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-17T09:42:09.492 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-17T09:42:09.492 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-17T09:42:09.492 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-17T09:42:09.492 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.compression_opts.enabled: false 2024-09-17T09:42:09.492 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-17T09:42:09.492 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-17T09:42:09.492 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-17T09:42:09.492 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-17T09:42:09.493 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.target_file_size_base: 67108864 2024-09-17T09:42:09.493 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-17T09:42:09.493 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-17T09:42:09.493 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-17T09:42:09.493 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-17T09:42:09.493 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-17T09:42:09.493 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-17T09:42:09.493 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-17T09:42:09.493 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-17T09:42:09.493 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-17T09:42:09.493 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-17T09:42:09.493 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-17T09:42:09.493 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-17T09:42:09.494 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-17T09:42:09.494 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-17T09:42:09.494 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.arena_block_size: 1048576 2024-09-17T09:42:09.494 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-17T09:42:09.494 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-17T09:42:09.494 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.disable_auto_compactions: 0 2024-09-17T09:42:09.494 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-17T09:42:09.494 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-17T09:42:09.494 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-17T09:42:09.494 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-17T09:42:09.494 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-17T09:42:09.494 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-17T09:42:09.494 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-17T09:42:09.494 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-17T09:42:09.495 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-17T09:42:09.495 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-17T09:42:09.495 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-17T09:42:09.495 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.inplace_update_support: 0 2024-09-17T09:42:09.495 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-17T09:42:09.495 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-17T09:42:09.495 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-17T09:42:09.495 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-17T09:42:09.495 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.bloom_locality: 0 2024-09-17T09:42:09.495 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.max_successive_merges: 0 2024-09-17T09:42:09.495 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-17T09:42:09.495 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.paranoid_file_checks: 0 2024-09-17T09:42:09.495 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.force_consistency_checks: 1 2024-09-17T09:42:09.495 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.report_bg_io_stats: 0 2024-09-17T09:42:09.496 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.ttl: 2592000 2024-09-17T09:42:09.496 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-17T09:42:09.496 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-17T09:42:09.496 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-17T09:42:09.496 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.enable_blob_files: false 2024-09-17T09:42:09.496 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.min_blob_size: 0 2024-09-17T09:42:09.496 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.blob_file_size: 268435456 2024-09-17T09:42:09.496 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-17T09:42:09.496 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-17T09:42:09.496 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-17T09:42:09.496 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-17T09:42:09.496 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-17T09:42:09.496 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.blob_file_starting_level: 0 2024-09-17T09:42:09.496 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-17T09:42:09.497 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 24.sst 2024-09-17T09:42:09.497 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-17T09:42:09.497 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi136/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 26, last_sequence is 12247, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-17T09:42:09.497 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2024-09-17T09:42:09.497 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 7d4968cd-73d2-4470-9108-0ea04685c7d3 2024-09-17T09:42:09.497 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726566129188690, "job": 1, "event": "recovery_started", "wal_files": [22]} 2024-09-17T09:42:09.497 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2024-09-17T09:42:09.497 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726566129206837, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 4747302, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12252, "largest_seqno": 13966, "table_properties": {"data_size": 4739705, "index_size": 4824, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1733, "raw_key_size": 15945, "raw_average_key_size": 23, "raw_value_size": 4724863, "raw_average_value_size": 6877, "num_data_blocks": 218, "num_entries": 687, "num_filter_entries": 687, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726566129, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "7d4968cd-73d2-4470-9108-0ea04685c7d3", "db_session_id": "1GMHPJHYL654PBB1FZPW", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2024-09-17T09:42:09.497 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726566129206999, "job": 1, "event": "recovery_finished"} 2024-09-17T09:42:09.497 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2024-09-17T09:42:09.497 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-17T09:42:09.497 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi136/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-17T09:42:09.497 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55d702d20e00 2024-09-17T09:42:09.498 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: DB pointer 0x55d702e22000 2024-09-17T09:42:09.498 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-17T09:42:09.498 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-17T09:42:09.498 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: ** DB Stats ** 2024-09-17T09:42:09.498 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T09:42:09.498 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-17T09:42:09.498 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T09:42:09.498 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T09:42:09.498 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-17T09:42:09.498 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T09:42:09.498 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T09:42:09.498 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: 2024-09-17T09:42:09.498 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: ** Compaction Stats [default] ** 2024-09-17T09:42:09.498 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T09:42:09.499 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-17T09:42:09.499 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: L0 1/0 4.53 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 632.9 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-17T09:42:09.499 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: L6 1/0 7.07 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-17T09:42:09.499 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: Sum 2/0 11.60 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 632.9 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-17T09:42:09.499 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 632.9 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-17T09:42:09.499 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: 2024-09-17T09:42:09.499 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: ** Compaction Stats [default] ** 2024-09-17T09:42:09.499 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T09:42:09.499 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T09:42:09.499 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 632.9 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-17T09:42:09.499 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: 2024-09-17T09:42:09.499 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-17T09:42:09.499 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: 2024-09-17T09:42:09.499 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T09:42:09.499 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: Flush(GB): cumulative 0.004, interval 0.004 2024-09-17T09:42:09.500 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T09:42:09.500 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T09:42:09.500 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T09:42:09.500 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-17T09:42:09.500 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: Cumulative compaction: 0.00 GB write, 195.26 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T09:42:09.500 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: Interval compaction: 0.00 GB write, 195.26 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T09:42:09.500 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T09:42:09.500 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: Block cache BinnedLRUCache@0x55d702cf1610#2 capacity: 512.00 MB usage: 6.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2024-09-17T09:42:09.500 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.78 KB,0.000339746%) IndexBlock(1,5.08 KB,0.000968575%) Misc(1,0.00 KB,0%) 2024-09-17T09:42:09.500 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: 2024-09-17T09:42:09.500 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-17T09:42:09.500 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: starting mon.smithi136 rank 1 at public addrs [v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0] at bind addrs [v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi136 fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 2024-09-17T09:42:09.500 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: mon.smithi136@-1(???) e2 preinit fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 2024-09-17T09:42:09.501 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: mon.smithi136@-1(???).mds e7 new map 2024-09-17T09:42:09.501 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: mon.smithi136@-1(???).mds e7 print_map 2024-09-17T09:42:09.501 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: e7 2024-09-17T09:42:09.501 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-17T09:42:09.501 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-17T09:42:09.501 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T09:42:09.501 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: legacy client fscid: 1 2024-09-17T09:42:09.501 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: 2024-09-17T09:42:09.501 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: Filesystem 'foofs' (1) 2024-09-17T09:42:09.501 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: fs_name foofs 2024-09-17T09:42:09.501 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: epoch 5 2024-09-17T09:42:09.501 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-17T09:42:09.501 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: created 2024-09-17T09:36:28.738531+0000 2024-09-17T09:42:09.501 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: modified 2024-09-17T09:36:35.535121+0000 2024-09-17T09:42:09.502 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: tableserver 0 2024-09-17T09:42:09.502 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: root 0 2024-09-17T09:42:09.502 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: session_timeout 60 2024-09-17T09:42:09.502 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: session_autoclose 300 2024-09-17T09:42:09.502 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: max_file_size 1099511627776 2024-09-17T09:42:09.502 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: max_xattr_size 65536 2024-09-17T09:42:09.502 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: required_client_features {} 2024-09-17T09:42:09.502 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: last_failure 0 2024-09-17T09:42:09.502 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: last_failure_osd_epoch 0 2024-09-17T09:42:09.502 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T09:42:09.502 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: max_mds 1 2024-09-17T09:42:09.502 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: in 0 2024-09-17T09:42:09.502 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: up {0=14486} 2024-09-17T09:42:09.502 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: failed 2024-09-17T09:42:09.503 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: damaged 2024-09-17T09:42:09.503 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: stopped 2024-09-17T09:42:09.503 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: data_pools [3] 2024-09-17T09:42:09.503 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: metadata_pool 2 2024-09-17T09:42:09.503 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: inline_data disabled 2024-09-17T09:42:09.503 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: balancer 2024-09-17T09:42:09.503 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: bal_rank_mask -1 2024-09-17T09:42:09.503 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: standby_count_wanted 1 2024-09-17T09:42:09.503 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: qdb_cluster leader: 0 members: 2024-09-17T09:42:09.503 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: [mds.foofs.smithi078.jdrsup{0:14486} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.78:6834/2954767313,v1:172.21.15.78:6835/2954767313] compat {c=[1],r=[1],i=[7ff]}] 2024-09-17T09:42:09.503 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: 2024-09-17T09:42:09.503 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: 2024-09-17T09:42:09.503 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: Standby daemons: 2024-09-17T09:42:09.503 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: 2024-09-17T09:42:09.503 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout: [mds.foofs.smithi136.smslff{-1:24265} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.136:6832/1260769482,v1:172.21.15.136:6833/1260769482] compat {c=[1],r=[1],i=[7ff]}] 2024-09-17T09:42:09.504 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: mon.smithi136@-1(???).osd e73 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-17T09:42:09.504 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: mon.smithi136@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T09:42:09.504 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: mon.smithi136@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T09:42:09.504 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: mon.smithi136@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T09:42:09.504 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 ceph-mon[49560]: mon.smithi136@-1(???).paxosservice(auth 1..30) refresh upgraded, format 0 -> 3 2024-09-17T09:42:09.504 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 bash[49542]: 4fce88842b79c82363aecf01ca6b4c6264d292d49e481ad99acacb35a9646d5a 2024-09-17T09:42:09.504 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:09 smithi136 systemd[1]: Started Ceph mon.smithi136 for 35c0cd22-74d7-11ef-bceb-c7b262605968. 2024-09-17T09:42:09.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 151612 11.40 MB/sec execute 184 sec latency 41.499 ms 2024-09-17T09:42:10.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 151987 11.41 MB/sec execute 185 sec latency 37.430 ms 2024-09-17T09:42:11.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:11 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mon metadata", "id": "smithi078"} : dispatch 2024-09-17T09:42:11.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:11 smithi078 ceph-mon[63787]: mon.smithi078 calling monitor election 2024-09-17T09:42:11.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:11 smithi078 ceph-mon[63787]: mon.smithi136 calling monitor election 2024-09-17T09:42:11.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:11 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mon metadata", "id": "smithi136"} : dispatch 2024-09-17T09:42:11.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:11 smithi078 ceph-mon[63787]: mon.smithi078 is new leader, mons smithi078,smithi136 in quorum (ranks 0,1) 2024-09-17T09:42:11.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:11 smithi078 ceph-mon[63787]: monmap epoch 3 2024-09-17T09:42:11.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:11 smithi078 ceph-mon[63787]: fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 2024-09-17T09:42:11.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:11 smithi078 ceph-mon[63787]: last_changed 2024-09-17T09:42:10.272627+0000 2024-09-17T09:42:11.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:11 smithi078 ceph-mon[63787]: created 2024-09-17T09:28:55.619057+0000 2024-09-17T09:42:11.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:11 smithi078 ceph-mon[63787]: min_mon_release 19 (squid) 2024-09-17T09:42:11.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:11 smithi078 ceph-mon[63787]: election_strategy: 1 2024-09-17T09:42:11.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:11 smithi078 ceph-mon[63787]: 0: [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon.smithi078 2024-09-17T09:42:11.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:11 smithi078 ceph-mon[63787]: 1: [v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0] mon.smithi136 2024-09-17T09:42:11.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:11 smithi078 ceph-mon[63787]: fsmap foofs:1 {0=foofs.smithi078.jdrsup=up:active} 1 up:standby 2024-09-17T09:42:11.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:11 smithi078 ceph-mon[63787]: osdmap e73: 8 total, 8 up, 8 in 2024-09-17T09:42:11.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:11 smithi078 ceph-mon[63787]: mgrmap e42: smithi078.rsqofi(active, since 17s), standbys: smithi136.hicmlm 2024-09-17T09:42:11.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:11 smithi078 ceph-mon[63787]: overall HEALTH_OK 2024-09-17T09:42:11.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:11 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:11.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:11 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:11.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:11 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:42:11.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:11 smithi078 ceph-mon[63787]: pgmap v12: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 5.3 MiB/s wr, 565 op/s 2024-09-17T09:42:11.597 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:11 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mon metadata", "id": "smithi078"} : dispatch 2024-09-17T09:42:11.597 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:11 smithi136 ceph-mon[49560]: mon.smithi078 calling monitor election 2024-09-17T09:42:11.597 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:11 smithi136 ceph-mon[49560]: mon.smithi136 calling monitor election 2024-09-17T09:42:11.597 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:11 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mon metadata", "id": "smithi136"} : dispatch 2024-09-17T09:42:11.597 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:11 smithi136 ceph-mon[49560]: mon.smithi078 is new leader, mons smithi078,smithi136 in quorum (ranks 0,1) 2024-09-17T09:42:11.597 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:11 smithi136 ceph-mon[49560]: monmap epoch 3 2024-09-17T09:42:11.598 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:11 smithi136 ceph-mon[49560]: fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 2024-09-17T09:42:11.598 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:11 smithi136 ceph-mon[49560]: last_changed 2024-09-17T09:42:10.272627+0000 2024-09-17T09:42:11.598 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:11 smithi136 ceph-mon[49560]: created 2024-09-17T09:28:55.619057+0000 2024-09-17T09:42:11.598 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:11 smithi136 ceph-mon[49560]: min_mon_release 19 (squid) 2024-09-17T09:42:11.598 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:11 smithi136 ceph-mon[49560]: election_strategy: 1 2024-09-17T09:42:11.598 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:11 smithi136 ceph-mon[49560]: 0: [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon.smithi078 2024-09-17T09:42:11.598 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:11 smithi136 ceph-mon[49560]: 1: [v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0] mon.smithi136 2024-09-17T09:42:11.598 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:11 smithi136 ceph-mon[49560]: fsmap foofs:1 {0=foofs.smithi078.jdrsup=up:active} 1 up:standby 2024-09-17T09:42:11.598 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:11 smithi136 ceph-mon[49560]: osdmap e73: 8 total, 8 up, 8 in 2024-09-17T09:42:11.598 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:11 smithi136 ceph-mon[49560]: mgrmap e42: smithi078.rsqofi(active, since 17s), standbys: smithi136.hicmlm 2024-09-17T09:42:11.598 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:11 smithi136 ceph-mon[49560]: overall HEALTH_OK 2024-09-17T09:42:11.598 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:11 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:11.598 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:11 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:11.598 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:11 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:42:11.598 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:11 smithi136 ceph-mon[49560]: pgmap v12: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 5.3 MiB/s wr, 565 op/s 2024-09-17T09:42:11.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 152318 11.40 MB/sec execute 186 sec latency 63.817 ms 2024-09-17T09:42:12.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 152535 11.38 MB/sec execute 187 sec latency 99.371 ms 2024-09-17T09:42:13.442 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:13 smithi078 ceph-mon[63787]: pgmap v13: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.9 MiB/s wr, 389 op/s 2024-09-17T09:42:13.678 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:13 smithi136 ceph-mon[49560]: pgmap v13: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.9 MiB/s wr, 389 op/s 2024-09-17T09:42:13.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 152736 11.35 MB/sec execute 188 sec latency 70.632 ms 2024-09-17T09:42:14.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 153049 11.32 MB/sec execute 189 sec latency 61.987 ms 2024-09-17T09:42:15.474 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:15 smithi078 ceph-mon[63787]: pgmap v14: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.0 MiB/s wr, 567 op/s 2024-09-17T09:42:15.475 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:15.475 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:15.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:15 smithi136 ceph-mon[49560]: pgmap v14: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.0 MiB/s wr, 567 op/s 2024-09-17T09:42:15.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:15.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:15.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 153502 11.32 MB/sec execute 190 sec latency 69.096 ms 2024-09-17T09:42:16.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:16 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:16.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:16 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:16.732 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:16 smithi078 ceph-mon[63787]: pgmap v15: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.7 MiB/s wr, 365 op/s 2024-09-17T09:42:16.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 154038 11.33 MB/sec execute 191 sec latency 41.347 ms 2024-09-17T09:42:16.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:16 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:16.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:16 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:16.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:16 smithi136 ceph-mon[49560]: pgmap v15: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.7 MiB/s wr, 365 op/s 2024-09-17T09:42:17.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 154512 11.30 MB/sec execute 192 sec latency 42.640 ms 2024-09-17T09:42:18.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 154868 11.30 MB/sec execute 193 sec latency 40.006 ms 2024-09-17T09:42:19.545 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:19 smithi078 ceph-mon[63787]: pgmap v16: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.7 MiB/s wr, 365 op/s 2024-09-17T09:42:19.545 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:19 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:19.545 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:19 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:19.545 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:19 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:19.545 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:19 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:42:19.545 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:19 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:19.545 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:19 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:19.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:19 smithi136 ceph-mon[49560]: pgmap v16: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.7 MiB/s wr, 365 op/s 2024-09-17T09:42:19.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:19 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:19.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:19 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:19.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:19 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:19.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:19 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:42:19.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:19 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:19.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:19 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:19.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 155298 11.33 MB/sec execute 194 sec latency 76.467 ms 2024-09-17T09:42:20.591 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:20 smithi078 ceph-mon[63787]: Reconfiguring mon.smithi078 (monmap changed)... 2024-09-17T09:42:20.591 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:20 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-17T09:42:20.591 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:20 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-17T09:42:20.591 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:20 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:20.591 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:20 smithi078 ceph-mon[63787]: Reconfiguring daemon mon.smithi078 on smithi078 2024-09-17T09:42:20.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:20 smithi136 ceph-mon[49560]: Reconfiguring mon.smithi078 (monmap changed)... 2024-09-17T09:42:20.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:20 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-17T09:42:20.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:20 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-17T09:42:20.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:20 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:20.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:20 smithi136 ceph-mon[49560]: Reconfiguring daemon mon.smithi078 on smithi078 2024-09-17T09:42:20.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 155726 11.35 MB/sec execute 195 sec latency 60.612 ms 2024-09-17T09:42:21.610 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:21 smithi078 ceph-mon[63787]: pgmap v17: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.0 MiB/s wr, 624 op/s 2024-09-17T09:42:21.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:21 smithi136 ceph-mon[49560]: pgmap v17: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.0 MiB/s wr, 624 op/s 2024-09-17T09:42:21.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 156018 11.34 MB/sec execute 196 sec latency 49.566 ms 2024-09-17T09:42:22.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 156280 11.32 MB/sec execute 197 sec latency 69.577 ms 2024-09-17T09:42:23.141 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:23 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:23.141 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:23 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:23.142 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:23 smithi078 ceph-mon[63787]: Reconfiguring mgr.smithi078.rsqofi (monmap changed)... 2024-09-17T09:42:23.142 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:23 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi078.rsqofi", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-17T09:42:23.142 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:23 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mgr services"} : dispatch 2024-09-17T09:42:23.142 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:23 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:23.142 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:23 smithi078 ceph-mon[63787]: Reconfiguring daemon mgr.smithi078.rsqofi on smithi078 2024-09-17T09:42:23.142 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:23 smithi078 ceph-mon[63787]: pgmap v18: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.5 MiB/s wr, 436 op/s 2024-09-17T09:42:23.142 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:23 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:42:23.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:23 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:23.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:23 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:23.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:23 smithi136 ceph-mon[49560]: Reconfiguring mgr.smithi078.rsqofi (monmap changed)... 2024-09-17T09:42:23.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:23 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi078.rsqofi", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-17T09:42:23.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:23 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mgr services"} : dispatch 2024-09-17T09:42:23.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:23 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:23.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:23 smithi136 ceph-mon[49560]: Reconfiguring daemon mgr.smithi078.rsqofi on smithi078 2024-09-17T09:42:23.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:23 smithi136 ceph-mon[49560]: pgmap v18: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.5 MiB/s wr, 436 op/s 2024-09-17T09:42:23.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:23 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:42:23.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 156541 11.28 MB/sec execute 198 sec latency 59.507 ms 2024-09-17T09:42:24.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 156796 11.26 MB/sec execute 199 sec latency 70.869 ms 2024-09-17T09:42:25.213 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:25 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:25.214 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:25 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:25.214 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:25 smithi078 ceph-mon[63787]: Reconfiguring crash.smithi078 (monmap changed)... 2024-09-17T09:42:25.214 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:25 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-17T09:42:25.214 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:25 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:25.214 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:25 smithi078 ceph-mon[63787]: Reconfiguring daemon crash.smithi078 on smithi078 2024-09-17T09:42:25.214 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:25 smithi078 ceph-mon[63787]: pgmap v19: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.8 MiB/s wr, 633 op/s 2024-09-17T09:42:25.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:25 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:25.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:25 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:25.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:25 smithi136 ceph-mon[49560]: Reconfiguring crash.smithi078 (monmap changed)... 2024-09-17T09:42:25.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:25 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-17T09:42:25.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:25 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:25.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:25 smithi136 ceph-mon[49560]: Reconfiguring daemon crash.smithi078 on smithi078 2024-09-17T09:42:25.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:25 smithi136 ceph-mon[49560]: pgmap v19: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.8 MiB/s wr, 633 op/s 2024-09-17T09:42:25.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 156982 11.23 MB/sec execute 200 sec latency 63.914 ms 2024-09-17T09:42:26.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 157243 11.21 MB/sec execute 201 sec latency 73.511 ms 2024-09-17T09:42:27.273 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:27 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:27.273 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:27 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:27.274 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:27 smithi078 ceph-mon[63787]: Reconfiguring osd.0 (monmap changed)... 2024-09-17T09:42:27.274 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:27 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-17T09:42:27.274 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:27 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:27.274 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:27 smithi078 ceph-mon[63787]: Reconfiguring daemon osd.0 on smithi078 2024-09-17T09:42:27.274 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:27 smithi078 ceph-mon[63787]: pgmap v20: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 455 op/s 2024-09-17T09:42:27.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:27 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:27.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:27 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:27.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:27 smithi136 ceph-mon[49560]: Reconfiguring osd.0 (monmap changed)... 2024-09-17T09:42:27.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:27 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-17T09:42:27.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:27 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:27.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:27 smithi136 ceph-mon[49560]: Reconfiguring daemon osd.0 on smithi078 2024-09-17T09:42:27.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:27 smithi136 ceph-mon[49560]: pgmap v20: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 455 op/s 2024-09-17T09:42:27.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 157594 11.19 MB/sec execute 202 sec latency 39.928 ms 2024-09-17T09:42:28.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 157954 11.15 MB/sec execute 203 sec latency 52.894 ms 2024-09-17T09:42:29.575 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:29 smithi078 ceph-mon[63787]: pgmap v21: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 455 op/s 2024-09-17T09:42:29.575 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:29 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:29.576 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:29 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:29.576 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:29 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-17T09:42:29.576 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:29 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:29.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:29 smithi136 ceph-mon[49560]: pgmap v21: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 455 op/s 2024-09-17T09:42:29.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:29 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:29.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:29 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:29.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:29 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-17T09:42:29.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:29 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:29.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 158292 11.14 MB/sec execute 204 sec latency 97.733 ms 2024-09-17T09:42:30.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:30 smithi078 ceph-mon[63787]: Reconfiguring osd.1 (monmap changed)... 2024-09-17T09:42:30.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:30 smithi078 ceph-mon[63787]: Reconfiguring daemon osd.1 on smithi078 2024-09-17T09:42:30.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:30 smithi136 ceph-mon[49560]: Reconfiguring osd.1 (monmap changed)... 2024-09-17T09:42:30.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:30 smithi136 ceph-mon[49560]: Reconfiguring daemon osd.1 on smithi078 2024-09-17T09:42:30.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 158647 11.15 MB/sec execute 205 sec latency 41.019 ms 2024-09-17T09:42:31.622 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:31 smithi078 ceph-mon[63787]: pgmap v22: 97 pgs: 97 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.0 MiB/s wr, 637 op/s 2024-09-17T09:42:31.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:31 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:31.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:31 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:31.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:31 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-17T09:42:31.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:31 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:31.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:31 smithi136 ceph-mon[49560]: pgmap v22: 97 pgs: 97 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.0 MiB/s wr, 637 op/s 2024-09-17T09:42:31.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:31 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:31.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:31 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:31.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:31 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-17T09:42:31.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:31 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:31.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 159110 11.18 MB/sec execute 206 sec latency 28.885 ms 2024-09-17T09:42:32.641 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:32 smithi078 ceph-mon[63787]: Reconfiguring osd.2 (monmap changed)... 2024-09-17T09:42:32.641 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:32 smithi078 ceph-mon[63787]: Reconfiguring daemon osd.2 on smithi078 2024-09-17T09:42:32.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:32 smithi136 ceph-mon[49560]: Reconfiguring osd.2 (monmap changed)... 2024-09-17T09:42:32.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:32 smithi136 ceph-mon[49560]: Reconfiguring daemon osd.2 on smithi078 2024-09-17T09:42:32.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 159604 11.20 MB/sec execute 207 sec latency 31.281 ms 2024-09-17T09:42:33.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:33 smithi078 ceph-mon[63787]: pgmap v23: 97 pgs: 97 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.7 MiB/s wr, 377 op/s 2024-09-17T09:42:33.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:33 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:33.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:33 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:33.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:33 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-17T09:42:33.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:33 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:33.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:33 smithi136 ceph-mon[49560]: pgmap v23: 97 pgs: 97 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.7 MiB/s wr, 377 op/s 2024-09-17T09:42:33.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:33 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:33.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:33 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:33.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:33 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-17T09:42:33.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:33 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:33.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 159888 11.18 MB/sec execute 208 sec latency 44.124 ms 2024-09-17T09:42:34.663 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:34 smithi078 ceph-mon[63787]: Reconfiguring osd.3 (monmap changed)... 2024-09-17T09:42:34.664 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:34 smithi078 ceph-mon[63787]: Reconfiguring daemon osd.3 on smithi078 2024-09-17T09:42:34.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:34 smithi136 ceph-mon[49560]: Reconfiguring osd.3 (monmap changed)... 2024-09-17T09:42:34.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:34 smithi136 ceph-mon[49560]: Reconfiguring daemon osd.3 on smithi078 2024-09-17T09:42:34.791 INFO:teuthology.orchestra.run.smithi078.stdout: 5 160159 11.15 MB/sec execute 209 sec latency 86.518 ms 2024-09-17T09:42:35.675 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:35 smithi078 ceph-mon[63787]: pgmap v24: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.5 MiB/s wr, 604 op/s 2024-09-17T09:42:35.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:35 smithi136 ceph-mon[49560]: pgmap v24: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.5 MiB/s wr, 604 op/s 2024-09-17T09:42:35.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 160417 11.13 MB/sec execute 210 sec latency 73.952 ms 2024-09-17T09:42:36.713 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:36 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:36.714 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:36 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:36.714 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:36 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.jdrsup", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-17T09:42:36.714 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:36 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:36.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 160687 11.11 MB/sec execute 211 sec latency 71.520 ms 2024-09-17T09:42:36.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:36 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:36.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:36 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:36.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:36 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.jdrsup", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-17T09:42:36.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:36 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:37.332 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-17T09:42:37.759 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:37 smithi078 ceph-mon[63787]: Reconfiguring mds.foofs.smithi078.jdrsup (monmap changed)... 2024-09-17T09:42:37.759 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:37 smithi078 ceph-mon[63787]: Reconfiguring daemon mds.foofs.smithi078.jdrsup on smithi078 2024-09-17T09:42:37.760 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:37 smithi078 ceph-mon[63787]: pgmap v25: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.1 MiB/s wr, 408 op/s 2024-09-17T09:42:37.760 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:37 smithi078 ceph-mon[63787]: from='client.34154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:42:37.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 161025 11.10 MB/sec execute 212 sec latency 72.759 ms 2024-09-17T09:42:37.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:37 smithi136 ceph-mon[49560]: Reconfiguring mds.foofs.smithi078.jdrsup (monmap changed)... 2024-09-17T09:42:37.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:37 smithi136 ceph-mon[49560]: Reconfiguring daemon mds.foofs.smithi078.jdrsup on smithi078 2024-09-17T09:42:37.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:37 smithi136 ceph-mon[49560]: pgmap v25: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.1 MiB/s wr, 408 op/s 2024-09-17T09:42:37.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:37 smithi136 ceph-mon[49560]: from='client.34154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:42:38.315 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:42:38.315 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (3m) 36s ago 12m 19.2M - 0.27.0 11f11916f8cd 97db2373ce92 2024-09-17T09:42:38.315 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (12m) 36s ago 12m 6400k - 17.2.0 e1d6a67b021e 46d44333624b 2024-09-17T09:42:38.315 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi136 smithi136 running (11m) 23s ago 11m 6395k - 17.2.0 e1d6a67b021e 56ce035cb482 2024-09-17T09:42:38.316 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (2m) 36s ago 11m 53.1M - dad864ee21e9 db74b0f8e0a7 2024-09-17T09:42:38.316 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.jdrsup smithi078 running (6m) 36s ago 6m 37.5M - 17.2.0 e1d6a67b021e 8f834e15ba88 2024-09-17T09:42:38.316 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi136.smslff smithi136 running (6m) 23s ago 6m 15.6M - 17.2.0 e1d6a67b021e 0c20e2b328d3 2024-09-17T09:42:38.316 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rsqofi smithi078 *:8443,9283,8765 running (2m) 36s ago 13m 488M - 19.3.0-4963-ge646f729 f271290a0950 7dda3108c9a0 2024-09-17T09:42:38.316 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi136.hicmlm smithi136 *:8443,9283,8765 running (75s) 23s ago 11m 452M - 19.3.0-4963-ge646f729 f271290a0950 fc518b58e07a 2024-09-17T09:42:38.316 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (56s) 36s ago 13m 32.4M 2048M 19.3.0-4963-ge646f729 f271290a0950 3ead101d3db1 2024-09-17T09:42:38.316 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi136 smithi136 running (29s) 23s ago 11m 24.6M 2048M 19.3.0-4963-ge646f729 f271290a0950 4fce88842b79 2024-09-17T09:42:38.316 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.tcwbvi smithi078 *:2049 running (5m) 36s ago 5m 264M - 4.0 e1d6a67b021e 30c87cb46555 2024-09-17T09:42:38.316 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi136.regjvc smithi136 *:2049 running (5m) 23s ago 5m 87.6M - 4.0 e1d6a67b021e b88e4dc24376 2024-09-17T09:42:38.316 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (2m) 36s ago 11m 11.4M - 1.7.0 72c9c2088986 52dad3adeeaa 2024-09-17T09:42:38.316 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi136 smithi136 *:9100 running (2m) 23s ago 11m 12.0M - 1.7.0 72c9c2088986 f2bc5856e35a 2024-09-17T09:42:38.316 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (10m) 36s ago 10m 498M 1936M 17.2.0 e1d6a67b021e b4ca42f8477d 2024-09-17T09:42:38.317 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (10m) 36s ago 10m 451M 1936M 17.2.0 e1d6a67b021e 351873014de4 2024-09-17T09:42:38.317 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (9m) 36s ago 9m 311M 1936M 17.2.0 e1d6a67b021e 07eb71e4fa57 2024-09-17T09:42:38.317 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (9m) 36s ago 9m 469M 1936M 17.2.0 e1d6a67b021e 1b3a8be89881 2024-09-17T09:42:38.317 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi136 running (8m) 23s ago 8m 507M 2704M 17.2.0 e1d6a67b021e a0c5b4de2414 2024-09-17T09:42:38.317 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi136 running (8m) 23s ago 8m 316M 2704M 17.2.0 e1d6a67b021e 30a3049955f9 2024-09-17T09:42:38.317 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi136 running (8m) 23s ago 7m 503M 2704M 17.2.0 e1d6a67b021e 1698e1567c33 2024-09-17T09:42:38.317 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi136 running (7m) 23s ago 7m 385M 2704M 17.2.0 e1d6a67b021e 5d3b08cecd78 2024-09-17T09:42:38.317 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (81s) 36s ago 11m 46.3M - 2.51.0 1d3b7f56885b 6e5e8c721eb0 2024-09-17T09:42:38.782 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:42:38.783 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:38 smithi078 ceph-mon[63787]: from='client.44103 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:42:38.783 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:38 smithi078 ceph-mon[63787]: from='client.34162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:42:38.783 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:38 smithi078 ceph-mon[63787]: pgmap v26: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.1 MiB/s wr, 408 op/s 2024-09-17T09:42:38.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 161394 11.07 MB/sec execute 213 sec latency 91.244 ms 2024-09-17T09:42:38.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:42:38.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:38 smithi136 ceph-mon[49560]: from='client.44103 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:42:38.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:38 smithi136 ceph-mon[49560]: from='client.34162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:42:38.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:38 smithi136 ceph-mon[49560]: pgmap v26: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.1 MiB/s wr, 408 op/s 2024-09-17T09:42:38.977 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:42:38.977 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-17T09:42:38.977 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:42:38.977 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:42:38.978 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-17T09:42:38.978 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:42:38.978 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:42:38.978 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-17T09:42:38.978 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T09:42:38.978 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:42:38.978 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-17T09:42:38.978 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:42:38.978 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:42:38.978 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-17T09:42:38.978 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-17T09:42:38.978 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 4 2024-09-17T09:42:38.978 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-17T09:42:38.979 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:42:39.500 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:42:39.500 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407", 2024-09-17T09:42:39.500 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-17T09:42:39.500 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T09:42:39.500 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-17T09:42:39.500 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-09-17T09:42:39.500 INFO:teuthology.orchestra.run.smithi078.stdout: "mon" 2024-09-17T09:42:39.501 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-17T09:42:39.501 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "4/23 daemons upgraded", 2024-09-17T09:42:39.501 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading mon daemons", 2024-09-17T09:42:39.501 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-17T09:42:39.501 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:42:39.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 161771 11.05 MB/sec execute 214 sec latency 53.815 ms 2024-09-17T09:42:39.835 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:39 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:39.836 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:39 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:39.836 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:39 smithi078 ceph-mon[63787]: Reconfiguring crash.smithi136 (monmap changed)... 2024-09-17T09:42:39.836 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:39 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi136", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-17T09:42:39.836 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:39 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:39.836 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:39 smithi078 ceph-mon[63787]: Reconfiguring daemon crash.smithi136 on smithi136 2024-09-17T09:42:39.836 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:39 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/3258649156' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:42:39.836 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:39 smithi078 ceph-mon[63787]: from='client.44111 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:42:39.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:39 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:39.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:39 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:39.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:39 smithi136 ceph-mon[49560]: Reconfiguring crash.smithi136 (monmap changed)... 2024-09-17T09:42:39.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:39 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi136", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-17T09:42:39.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:39 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:39.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:39 smithi136 ceph-mon[49560]: Reconfiguring daemon crash.smithi136 on smithi136 2024-09-17T09:42:39.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:39 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/3258649156' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:42:39.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:39 smithi136 ceph-mon[49560]: from='client.44111 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:42:40.126 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-17T09:42:40.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 162081 11.05 MB/sec execute 215 sec latency 37.310 ms 2024-09-17T09:42:40.900 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:40 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/748652101' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:42:40.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:40 smithi078 ceph-mon[63787]: pgmap v27: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.7 MiB/s wr, 601 op/s 2024-09-17T09:42:40.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:40 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:40.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:40 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:40.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:40 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi136.hicmlm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-17T09:42:40.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:40 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mgr services"} : dispatch 2024-09-17T09:42:40.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:40 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:41.025 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:40 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/748652101' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:42:41.026 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:40 smithi136 ceph-mon[49560]: pgmap v27: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.7 MiB/s wr, 601 op/s 2024-09-17T09:42:41.026 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:40 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:41.026 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:40 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:41.026 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:40 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi136.hicmlm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-17T09:42:41.026 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:40 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mgr services"} : dispatch 2024-09-17T09:42:41.026 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:40 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:41.792 INFO:teuthology.orchestra.run.smithi078.stdout: 5 162504 11.09 MB/sec execute 216 sec latency 55.661 ms 2024-09-17T09:42:41.914 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:41 smithi078 ceph-mon[63787]: Reconfiguring mgr.smithi136.hicmlm (monmap changed)... 2024-09-17T09:42:41.914 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:41 smithi078 ceph-mon[63787]: Reconfiguring daemon mgr.smithi136.hicmlm on smithi136 2024-09-17T09:42:41.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:41 smithi136 ceph-mon[49560]: Reconfiguring mgr.smithi136.hicmlm (monmap changed)... 2024-09-17T09:42:41.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:41 smithi136 ceph-mon[49560]: Reconfiguring daemon mgr.smithi136.hicmlm on smithi136 2024-09-17T09:42:42.801 INFO:teuthology.orchestra.run.smithi078.stdout: 5 162880 11.08 MB/sec execute 217 sec latency 48.119 ms 2024-09-17T09:42:43.182 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:43 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:43.182 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:43 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:43.183 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:43 smithi078 ceph-mon[63787]: Reconfiguring mon.smithi136 (monmap changed)... 2024-09-17T09:42:43.183 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:43 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-17T09:42:43.183 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:43 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-17T09:42:43.183 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:43 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:43.183 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:43 smithi078 ceph-mon[63787]: Reconfiguring daemon mon.smithi136 on smithi136 2024-09-17T09:42:43.183 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:43 smithi078 ceph-mon[63787]: pgmap v28: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.3 MiB/s wr, 419 op/s 2024-09-17T09:42:43.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:43 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:43.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:43 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:43.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:43 smithi136 ceph-mon[49560]: Reconfiguring mon.smithi136 (monmap changed)... 2024-09-17T09:42:43.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:43 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-17T09:42:43.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:43 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-17T09:42:43.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:43 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:43.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:43 smithi136 ceph-mon[49560]: Reconfiguring daemon mon.smithi136 on smithi136 2024-09-17T09:42:43.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:43 smithi136 ceph-mon[49560]: pgmap v28: 97 pgs: 97 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.3 MiB/s wr, 419 op/s 2024-09-17T09:42:43.801 INFO:teuthology.orchestra.run.smithi078.stdout: 5 163152 11.08 MB/sec execute 218 sec latency 64.377 ms 2024-09-17T09:42:44.725 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:44 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:44.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:44 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:44.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:44 smithi078 ceph-mon[63787]: Reconfiguring osd.4 (monmap changed)... 2024-09-17T09:42:44.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:44 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-17T09:42:44.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:44 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:44.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:44 smithi078 ceph-mon[63787]: Reconfiguring daemon osd.4 on smithi136 2024-09-17T09:42:44.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:44 smithi078 ceph-mon[63787]: pgmap v29: 97 pgs: 97 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.7 MiB/s wr, 625 op/s 2024-09-17T09:42:44.801 INFO:teuthology.orchestra.run.smithi078.stdout: 5 163396 11.06 MB/sec execute 219 sec latency 66.265 ms 2024-09-17T09:42:44.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:44 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:44.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:44 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:44.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:44 smithi136 ceph-mon[49560]: Reconfiguring osd.4 (monmap changed)... 2024-09-17T09:42:44.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:44 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-17T09:42:44.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:44 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:44.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:44 smithi136 ceph-mon[49560]: Reconfiguring daemon osd.4 on smithi136 2024-09-17T09:42:44.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:44 smithi136 ceph-mon[49560]: pgmap v29: 97 pgs: 97 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.7 MiB/s wr, 625 op/s 2024-09-17T09:42:45.801 INFO:teuthology.orchestra.run.smithi078.stdout: 5 163935 11.05 MB/sec execute 220 sec latency 33.854 ms 2024-09-17T09:42:46.767 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:46.767 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:46.768 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:46 smithi078 ceph-mon[63787]: Reconfiguring osd.5 (monmap changed)... 2024-09-17T09:42:46.768 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-17T09:42:46.768 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:46.768 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:46 smithi078 ceph-mon[63787]: Reconfiguring daemon osd.5 on smithi136 2024-09-17T09:42:46.768 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:46 smithi078 ceph-mon[63787]: pgmap v30: 97 pgs: 97 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.0 MiB/s wr, 398 op/s 2024-09-17T09:42:46.801 INFO:teuthology.orchestra.run.smithi078.stdout: 5 164555 11.08 MB/sec execute 221 sec latency 21.815 ms 2024-09-17T09:42:46.961 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:46.961 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:46.961 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:46 smithi136 ceph-mon[49560]: Reconfiguring osd.5 (monmap changed)... 2024-09-17T09:42:46.961 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-17T09:42:46.961 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:46.961 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:46 smithi136 ceph-mon[49560]: Reconfiguring daemon osd.5 on smithi136 2024-09-17T09:42:46.961 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:46 smithi136 ceph-mon[49560]: pgmap v30: 97 pgs: 97 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.0 MiB/s wr, 398 op/s 2024-09-17T09:42:47.801 INFO:teuthology.orchestra.run.smithi078.stdout: 5 165351 11.09 MB/sec execute 222 sec latency 67.268 ms 2024-09-17T09:42:48.273 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:48 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:48.274 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:48 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:48.274 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:48 smithi078 ceph-mon[63787]: Reconfiguring osd.6 (monmap changed)... 2024-09-17T09:42:48.274 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:48 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-17T09:42:48.274 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:48 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:48.274 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:48 smithi078 ceph-mon[63787]: Reconfiguring daemon osd.6 on smithi136 2024-09-17T09:42:48.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:48 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:48.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:48 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:48.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:48 smithi136 ceph-mon[49560]: Reconfiguring osd.6 (monmap changed)... 2024-09-17T09:42:48.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:48 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-17T09:42:48.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:48 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:48.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:48 smithi136 ceph-mon[49560]: Reconfiguring daemon osd.6 on smithi136 2024-09-17T09:42:48.801 INFO:teuthology.orchestra.run.smithi078.stdout: 5 166125 11.17 MB/sec execute 223 sec latency 19.172 ms 2024-09-17T09:42:49.318 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:49 smithi136 ceph-mon[49560]: pgmap v31: 97 pgs: 97 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.0 MiB/s wr, 398 op/s 2024-09-17T09:42:49.318 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:49 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:49.318 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:49 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:49.318 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:49 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-17T09:42:49.318 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:49 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:49.323 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:49 smithi078 ceph-mon[63787]: pgmap v31: 97 pgs: 97 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.0 MiB/s wr, 398 op/s 2024-09-17T09:42:49.323 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:49 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:49.324 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:49 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:49.324 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:49 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-17T09:42:49.324 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:49 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:49.801 INFO:teuthology.orchestra.run.smithi078.stdout: 5 166636 11.20 MB/sec execute 224 sec latency 72.555 ms 2024-09-17T09:42:50.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:50 smithi136 ceph-mon[49560]: Reconfiguring osd.7 (monmap changed)... 2024-09-17T09:42:50.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:50 smithi136 ceph-mon[49560]: Reconfiguring daemon osd.7 on smithi136 2024-09-17T09:42:50.260 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:50 smithi078 ceph-mon[63787]: Reconfiguring osd.7 (monmap changed)... 2024-09-17T09:42:50.260 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:50 smithi078 ceph-mon[63787]: Reconfiguring daemon osd.7 on smithi136 2024-09-17T09:42:50.801 INFO:teuthology.orchestra.run.smithi078.stdout: 5 167053 11.20 MB/sec execute 225 sec latency 45.810 ms 2024-09-17T09:42:51.341 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:51 smithi078 ceph-mon[63787]: pgmap v32: 97 pgs: 97 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 735 op/s 2024-09-17T09:42:51.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:51 smithi136 ceph-mon[49560]: pgmap v32: 97 pgs: 97 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 735 op/s 2024-09-17T09:42:51.801 INFO:teuthology.orchestra.run.smithi078.stdout: 5 167342 11.17 MB/sec execute 226 sec latency 58.136 ms 2024-09-17T09:42:52.351 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:52.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:52.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:52 smithi078 ceph-mon[63787]: Reconfiguring mds.foofs.smithi136.smslff (monmap changed)... 2024-09-17T09:42:52.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi136.smslff", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-17T09:42:52.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:52.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:52 smithi078 ceph-mon[63787]: Reconfiguring daemon mds.foofs.smithi136.smslff on smithi136 2024-09-17T09:42:52.720 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:52 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:52.721 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:52 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:52.721 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:52 smithi136 ceph-mon[49560]: Reconfiguring mds.foofs.smithi136.smslff (monmap changed)... 2024-09-17T09:42:52.721 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:52 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi136.smslff", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-17T09:42:52.721 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:52 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:52.721 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:52 smithi136 ceph-mon[49560]: Reconfiguring daemon mds.foofs.smithi136.smslff on smithi136 2024-09-17T09:42:52.801 INFO:teuthology.orchestra.run.smithi078.stdout: 5 167634 11.16 MB/sec execute 227 sec latency 79.485 ms 2024-09-17T09:42:53.387 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:53 smithi078 ceph-mon[63787]: pgmap v33: 97 pgs: 97 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.7 MiB/s wr, 543 op/s 2024-09-17T09:42:53.387 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:53 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:42:53.387 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:53 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:53.387 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:53 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:53.387 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:53 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:42:53.387 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:53 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:42:53.387 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:53 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:42:53.387 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:53 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:53.388 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:53 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi078"} : dispatch 2024-09-17T09:42:53.388 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:53 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi078"}]': finished 2024-09-17T09:42:53.388 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:53 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi136"} : dispatch 2024-09-17T09:42:53.388 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:53 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi136"}]': finished 2024-09-17T09:42:53.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:53 smithi136 ceph-mon[49560]: pgmap v33: 97 pgs: 97 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.7 MiB/s wr, 543 op/s 2024-09-17T09:42:53.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:53 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:42:53.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:53 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:53.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:53 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:53.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:53 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:42:53.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:53 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:42:53.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:53 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:42:53.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:53 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:53.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:53 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi078"} : dispatch 2024-09-17T09:42:53.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:53 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi078"}]': finished 2024-09-17T09:42:53.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:53 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi136"} : dispatch 2024-09-17T09:42:53.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:53 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi136"}]': finished 2024-09-17T09:42:53.801 INFO:teuthology.orchestra.run.smithi078.stdout: 5 167912 11.14 MB/sec execute 228 sec latency 70.321 ms 2024-09-17T09:42:54.256 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:54 smithi078 ceph-mon[63787]: Upgrade: Setting container_image for all mon 2024-09-17T09:42:54.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:54 smithi136 ceph-mon[49560]: Upgrade: Setting container_image for all mon 2024-09-17T09:42:54.801 INFO:teuthology.orchestra.run.smithi078.stdout: 5 168316 11.13 MB/sec execute 229 sec latency 63.153 ms 2024-09-17T09:42:55.285 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:55 smithi078 ceph-mon[63787]: pgmap v34: 97 pgs: 97 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 788 op/s 2024-09-17T09:42:55.286 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:55 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:55.286 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:55 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-17T09:42:55.286 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:55 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:55.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:55 smithi136 ceph-mon[49560]: pgmap v34: 97 pgs: 97 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.2 MiB/s wr, 788 op/s 2024-09-17T09:42:55.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:55 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:42:55.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:55 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-17T09:42:55.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:55 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:42:55.801 INFO:teuthology.orchestra.run.smithi078.stdout: 5 168628 11.09 MB/sec execute 230 sec latency 59.083 ms 2024-09-17T09:42:56.330 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:56 smithi078 ceph-mon[63787]: Upgrade: Updating crash.smithi078 (1/2) 2024-09-17T09:42:56.331 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:56 smithi078 ceph-mon[63787]: Deploying daemon crash.smithi078 on smithi078 2024-09-17T09:42:56.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:56 smithi136 ceph-mon[49560]: Upgrade: Updating crash.smithi078 (1/2) 2024-09-17T09:42:56.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:56 smithi136 ceph-mon[49560]: Deploying daemon crash.smithi078 on smithi078 2024-09-17T09:42:56.801 INFO:teuthology.orchestra.run.smithi078.stdout: 5 168993 11.09 MB/sec execute 231 sec latency 34.443 ms 2024-09-17T09:42:57.424 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:57 smithi078 ceph-mon[63787]: pgmap v35: 97 pgs: 97 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.8 MiB/s wr, 582 op/s 2024-09-17T09:42:57.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:57 smithi136 ceph-mon[49560]: pgmap v35: 97 pgs: 97 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.8 MiB/s wr, 582 op/s 2024-09-17T09:42:57.802 INFO:teuthology.orchestra.run.smithi078.stdout: 5 169247 11.09 MB/sec execute 232 sec latency 139.199 ms 2024-09-17T09:42:58.802 INFO:teuthology.orchestra.run.smithi078.stdout: 5 169634 11.10 MB/sec execute 233 sec latency 46.678 ms 2024-09-17T09:42:59.472 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:42:59 smithi078 ceph-mon[63787]: pgmap v36: 97 pgs: 97 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.8 MiB/s wr, 582 op/s 2024-09-17T09:42:59.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:42:59 smithi136 ceph-mon[49560]: pgmap v36: 97 pgs: 97 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.8 MiB/s wr, 582 op/s 2024-09-17T09:42:59.802 INFO:teuthology.orchestra.run.smithi078.stdout: 5 169961 11.09 MB/sec execute 234 sec latency 56.384 ms 2024-09-17T09:43:00.810 INFO:teuthology.orchestra.run.smithi078.stdout: 5 170204 11.09 MB/sec execute 235 sec latency 69.303 ms 2024-09-17T09:43:01.525 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:01 smithi078 ceph-mon[63787]: pgmap v37: 97 pgs: 97 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 784 op/s 2024-09-17T09:43:01.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:01 smithi136 ceph-mon[49560]: pgmap v37: 97 pgs: 97 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 784 op/s 2024-09-17T09:43:01.810 INFO:teuthology.orchestra.run.smithi078.stdout: 5 170492 11.07 MB/sec execute 236 sec latency 45.988 ms 2024-09-17T09:43:02.544 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:02 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:02.545 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:02 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:02.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:02 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:02.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:02 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:02.809 INFO:teuthology.orchestra.run.smithi078.stdout: 5 170811 11.05 MB/sec execute 237 sec latency 43.090 ms 2024-09-17T09:43:03.551 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:03 smithi078 ceph-mon[63787]: pgmap v38: 97 pgs: 97 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.4 MiB/s wr, 446 op/s 2024-09-17T09:43:03.552 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:03 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:03.552 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:03 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi136", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-17T09:43:03.552 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:03 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:43:03.650 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:03 smithi136 ceph-mon[49560]: pgmap v38: 97 pgs: 97 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.4 MiB/s wr, 446 op/s 2024-09-17T09:43:03.650 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:03 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:03.650 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:03 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi136", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-17T09:43:03.650 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:03 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:43:03.810 INFO:teuthology.orchestra.run.smithi078.stdout: 5 171139 11.04 MB/sec execute 238 sec latency 54.894 ms 2024-09-17T09:43:04.567 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:04 smithi078 ceph-mon[63787]: Upgrade: Updating crash.smithi136 (2/2) 2024-09-17T09:43:04.568 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:04 smithi078 ceph-mon[63787]: Deploying daemon crash.smithi136 on smithi136 2024-09-17T09:43:04.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:04 smithi136 ceph-mon[49560]: Upgrade: Updating crash.smithi136 (2/2) 2024-09-17T09:43:04.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:04 smithi136 ceph-mon[49560]: Deploying daemon crash.smithi136 on smithi136 2024-09-17T09:43:04.809 INFO:teuthology.orchestra.run.smithi078.stdout: 5 171357 11.02 MB/sec execute 239 sec latency 63.440 ms 2024-09-17T09:43:05.618 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:05 smithi078 ceph-mon[63787]: pgmap v39: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 633 op/s 2024-09-17T09:43:05.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:05 smithi136 ceph-mon[49560]: pgmap v39: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 633 op/s 2024-09-17T09:43:05.810 INFO:teuthology.orchestra.run.smithi078.stdout: 5 171653 10.98 MB/sec execute 240 sec latency 69.881 ms 2024-09-17T09:43:06.810 INFO:teuthology.orchestra.run.smithi078.stdout: 5 171956 10.97 MB/sec execute 241 sec latency 74.659 ms 2024-09-17T09:43:07.664 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:07 smithi078 ceph-mon[63787]: pgmap v40: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.0 MiB/s wr, 388 op/s 2024-09-17T09:43:07.665 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:07 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:07.665 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:07 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:07.665 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:07 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:43:07.810 INFO:teuthology.orchestra.run.smithi078.stdout: 5 172205 10.94 MB/sec execute 242 sec latency 75.400 ms 2024-09-17T09:43:07.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:07 smithi136 ceph-mon[49560]: pgmap v40: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.0 MiB/s wr, 388 op/s 2024-09-17T09:43:07.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:07 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:07.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:07 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:07.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:07 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:43:08.704 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:08 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:43:08.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:08 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:43:08.810 INFO:teuthology.orchestra.run.smithi078.stdout: 5 172574 10.95 MB/sec execute 243 sec latency 42.555 ms 2024-09-17T09:43:09.714 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:09 smithi078 ceph-mon[63787]: pgmap v41: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.0 MiB/s wr, 388 op/s 2024-09-17T09:43:09.810 INFO:teuthology.orchestra.run.smithi078.stdout: 5 173079 10.98 MB/sec execute 244 sec latency 27.674 ms 2024-09-17T09:43:09.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:09 smithi136 ceph-mon[49560]: pgmap v41: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.0 MiB/s wr, 388 op/s 2024-09-17T09:43:10.736 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-17T09:43:10.740 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:10 smithi078 ceph-mon[63787]: pgmap v42: 97 pgs: 97 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.3 MiB/s wr, 568 op/s 2024-09-17T09:43:10.741 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:10 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:10.741 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:10 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:10.810 INFO:teuthology.orchestra.run.smithi078.stdout: 5 173496 10.98 MB/sec execute 245 sec latency 45.183 ms 2024-09-17T09:43:10.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:10 smithi136 ceph-mon[49560]: pgmap v42: 97 pgs: 97 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.3 MiB/s wr, 568 op/s 2024-09-17T09:43:10.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:10 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:10.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:10 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:11.810 INFO:teuthology.orchestra.run.smithi078.stdout: 5 173764 10.98 MB/sec execute 246 sec latency 60.338 ms 2024-09-17T09:43:11.954 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:43:11.954 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (3m) 70s ago 12m 19.2M - 0.27.0 11f11916f8cd 97db2373ce92 2024-09-17T09:43:11.954 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 starting - - - - 2024-09-17T09:43:11.954 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi136 smithi136 running (5s) 1s ago 12m 6899k - 19.3.0-4963-ge646f729 f271290a0950 909f405348cc 2024-09-17T09:43:11.954 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (3m) 70s ago 12m 53.1M - dad864ee21e9 db74b0f8e0a7 2024-09-17T09:43:11.954 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.jdrsup smithi078 running (6m) 70s ago 6m 37.5M - 17.2.0 e1d6a67b021e 8f834e15ba88 2024-09-17T09:43:11.954 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi136.smslff smithi136 running (6m) 1s ago 6m 15.8M - 17.2.0 e1d6a67b021e 0c20e2b328d3 2024-09-17T09:43:11.954 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rsqofi smithi078 *:8443,9283,8765 running (2m) 70s ago 13m 488M - 19.3.0-4963-ge646f729 f271290a0950 7dda3108c9a0 2024-09-17T09:43:11.954 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi136.hicmlm smithi136 *:8443,9283,8765 running (108s) 1s ago 12m 453M - 19.3.0-4963-ge646f729 f271290a0950 fc518b58e07a 2024-09-17T09:43:11.954 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (90s) 70s ago 14m 32.4M 2048M 19.3.0-4963-ge646f729 f271290a0950 3ead101d3db1 2024-09-17T09:43:11.954 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi136 smithi136 running (63s) 1s ago 11m 36.0M 2048M 19.3.0-4963-ge646f729 f271290a0950 4fce88842b79 2024-09-17T09:43:11.955 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.tcwbvi smithi078 *:2049 running (6m) 70s ago 6m 264M - 4.0 e1d6a67b021e 30c87cb46555 2024-09-17T09:43:11.955 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi136.regjvc smithi136 *:2049 running (6m) 1s ago 6m 87.7M - 4.0 e1d6a67b021e b88e4dc24376 2024-09-17T09:43:11.955 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (3m) 70s ago 12m 11.4M - 1.7.0 72c9c2088986 52dad3adeeaa 2024-09-17T09:43:11.955 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi136 smithi136 *:9100 running (2m) 1s ago 11m 11.9M - 1.7.0 72c9c2088986 f2bc5856e35a 2024-09-17T09:43:11.955 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (11m) 70s ago 11m 498M 1936M 17.2.0 e1d6a67b021e b4ca42f8477d 2024-09-17T09:43:11.955 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (10m) 70s ago 10m 451M 1936M 17.2.0 e1d6a67b021e 351873014de4 2024-09-17T09:43:11.955 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (10m) 70s ago 10m 311M 1936M 17.2.0 e1d6a67b021e 07eb71e4fa57 2024-09-17T09:43:11.955 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (9m) 70s ago 9m 469M 1936M 17.2.0 e1d6a67b021e 1b3a8be89881 2024-09-17T09:43:11.955 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi136 running (9m) 1s ago 9m 557M 2704M 17.2.0 e1d6a67b021e a0c5b4de2414 2024-09-17T09:43:11.955 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi136 running (8m) 1s ago 8m 383M 2704M 17.2.0 e1d6a67b021e 30a3049955f9 2024-09-17T09:43:11.955 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi136 running (8m) 1s ago 8m 572M 2704M 17.2.0 e1d6a67b021e 1698e1567c33 2024-09-17T09:43:11.955 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi136 running (8m) 1s ago 8m 453M 2704M 17.2.0 e1d6a67b021e 5d3b08cecd78 2024-09-17T09:43:11.955 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (115s) 70s ago 11m 46.3M - 2.51.0 1d3b7f56885b 6e5e8c721eb0 2024-09-17T09:43:12.270 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:12 smithi078 ceph-mon[63787]: from='client.34178 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:43:12.270 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:12 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:12.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:12 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:12.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:12 smithi078 ceph-mon[63787]: from='client.34182 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:43:12.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:12 smithi136 ceph-mon[49560]: from='client.34178 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:43:12.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:12 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:12.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:12 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:12.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:12 smithi136 ceph-mon[49560]: from='client.34182 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:43:12.693 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:43:12.693 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-17T09:43:12.693 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:43:12.693 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:43:12.694 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-17T09:43:12.694 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:43:12.694 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:43:12.694 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-17T09:43:12.694 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T09:43:12.694 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:43:12.694 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-17T09:43:12.694 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:43:12.694 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:43:12.694 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-17T09:43:12.694 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-17T09:43:12.694 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 4 2024-09-17T09:43:12.695 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-17T09:43:12.695 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:43:12.810 INFO:teuthology.orchestra.run.smithi078.stdout: 5 174026 10.97 MB/sec execute 247 sec latency 48.002 ms 2024-09-17T09:43:13.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:13 smithi136 ceph-mon[49560]: from='client.44123 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:43:13.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:13 smithi136 ceph-mon[49560]: pgmap v43: 97 pgs: 97 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.5 MiB/s wr, 366 op/s 2024-09-17T09:43:13.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:13 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/3757534384' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:43:13.292 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:13 smithi078 ceph-mon[63787]: from='client.44123 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:43:13.292 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:13 smithi078 ceph-mon[63787]: pgmap v43: 97 pgs: 97 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.5 MiB/s wr, 366 op/s 2024-09-17T09:43:13.292 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:13 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/3757534384' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:43:13.315 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:43:13.315 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407", 2024-09-17T09:43:13.315 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-17T09:43:13.315 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T09:43:13.315 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-17T09:43:13.316 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-09-17T09:43:13.316 INFO:teuthology.orchestra.run.smithi078.stdout: "mon" 2024-09-17T09:43:13.316 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-17T09:43:13.316 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "5/23 daemons upgraded", 2024-09-17T09:43:13.316 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading crash daemons", 2024-09-17T09:43:13.316 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-17T09:43:13.316 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:43:13.810 INFO:teuthology.orchestra.run.smithi078.stdout: 5 174270 10.94 MB/sec execute 248 sec latency 51.591 ms 2024-09-17T09:43:14.056 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-17T09:43:14.303 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:14 smithi078 ceph-mon[63787]: from='client.44129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:43:14.303 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:14 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/61948100' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:43:14.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:14 smithi136 ceph-mon[49560]: from='client.44129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:43:14.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:14 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/61948100' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:43:14.810 INFO:teuthology.orchestra.run.smithi078.stdout: 5 174548 10.92 MB/sec execute 249 sec latency 78.304 ms 2024-09-17T09:43:15.362 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:15 smithi078 ceph-mon[63787]: pgmap v44: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.1 MiB/s wr, 574 op/s 2024-09-17T09:43:15.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:15.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:15.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:15 smithi136 ceph-mon[49560]: pgmap v44: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.1 MiB/s wr, 574 op/s 2024-09-17T09:43:15.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:15.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:15.810 INFO:teuthology.orchestra.run.smithi078.stdout: 5 174815 10.90 MB/sec execute 250 sec latency 43.985 ms 2024-09-17T09:43:16.810 INFO:teuthology.orchestra.run.smithi078.stdout: 5 175139 10.88 MB/sec execute 251 sec latency 58.451 ms 2024-09-17T09:43:17.664 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:17 smithi078 ceph-mon[63787]: pgmap v45: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.9 MiB/s wr, 387 op/s 2024-09-17T09:43:17.665 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:17 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:17.665 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:17 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:17.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:17 smithi136 ceph-mon[49560]: pgmap v45: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.9 MiB/s wr, 387 op/s 2024-09-17T09:43:17.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:17 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:17.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:17 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:17.810 INFO:teuthology.orchestra.run.smithi078.stdout: 5 175485 10.87 MB/sec execute 252 sec latency 52.653 ms 2024-09-17T09:43:18.810 INFO:teuthology.orchestra.run.smithi078.stdout: 5 175799 10.84 MB/sec execute 253 sec latency 59.777 ms 2024-09-17T09:43:18.979 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:18.979 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:18.979 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:18 smithi078 ceph-mon[63787]: pgmap v46: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.9 MiB/s wr, 387 op/s 2024-09-17T09:43:19.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:19.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:19.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:18 smithi136 ceph-mon[49560]: pgmap v46: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.9 MiB/s wr, 387 op/s 2024-09-17T09:43:19.810 INFO:teuthology.orchestra.run.smithi078.stdout: 5 176232 10.86 MB/sec execute 254 sec latency 47.096 ms 2024-09-17T09:43:20.810 INFO:teuthology.orchestra.run.smithi078.stdout: 5 176750 10.89 MB/sec execute 255 sec latency 47.336 ms 2024-09-17T09:43:21.505 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:21 smithi078 ceph-mon[63787]: pgmap v47: 97 pgs: 97 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.4 MiB/s wr, 578 op/s 2024-09-17T09:43:21.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:21 smithi136 ceph-mon[49560]: pgmap v47: 97 pgs: 97 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.4 MiB/s wr, 578 op/s 2024-09-17T09:43:21.810 INFO:teuthology.orchestra.run.smithi078.stdout: 5 177187 10.90 MB/sec execute 256 sec latency 56.193 ms 2024-09-17T09:43:22.810 INFO:teuthology.orchestra.run.smithi078.stdout: 5 177560 10.91 MB/sec execute 257 sec latency 48.871 ms 2024-09-17T09:43:23.549 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:23 smithi078 ceph-mon[63787]: pgmap v48: 97 pgs: 97 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.1 MiB/s wr, 398 op/s 2024-09-17T09:43:23.549 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:23 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:43:23.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:23 smithi136 ceph-mon[49560]: pgmap v48: 97 pgs: 97 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.1 MiB/s wr, 398 op/s 2024-09-17T09:43:23.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:23 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:43:23.813 INFO:teuthology.orchestra.run.smithi078.stdout: 5 177877 10.89 MB/sec execute 258 sec latency 66.173 ms 2024-09-17T09:43:24.813 INFO:teuthology.orchestra.run.smithi078.stdout: 5 178219 10.88 MB/sec execute 259 sec latency 107.933 ms 2024-09-17T09:43:25.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:25 smithi078 ceph-mon[63787]: pgmap v49: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 8.6 KiB/s rd, 7.0 MiB/s wr, 644 op/s 2024-09-17T09:43:25.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:25 smithi078 ceph-mon[63787]: mgrmap e43: smithi078.rsqofi(active, since 92s), standbys: smithi136.hicmlm 2024-09-17T09:43:25.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:25 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:25.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:25 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:25.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:25 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:43:25.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:25 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:43:25.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:25 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:25.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:25 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:25.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:25 smithi136 ceph-mon[49560]: pgmap v49: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 8.6 KiB/s rd, 7.0 MiB/s wr, 644 op/s 2024-09-17T09:43:25.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:25 smithi136 ceph-mon[49560]: mgrmap e43: smithi078.rsqofi(active, since 92s), standbys: smithi136.hicmlm 2024-09-17T09:43:25.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:25 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:25.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:25 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:25.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:25 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:43:25.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:25 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:43:25.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:25 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:25.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:25 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:25.813 INFO:teuthology.orchestra.run.smithi078.stdout: 5 178557 10.87 MB/sec execute 260 sec latency 71.947 ms 2024-09-17T09:43:26.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:26 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:43:26.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:26 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:43:26.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:26 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:43:26.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:26 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:43:26.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:26 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:26.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:26 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi078"} : dispatch 2024-09-17T09:43:26.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:26 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi078"}]': finished 2024-09-17T09:43:26.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:26 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi136"} : dispatch 2024-09-17T09:43:26.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:26 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi136"}]': finished 2024-09-17T09:43:26.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:26 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-17T09:43:26.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:26 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:43:26.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:26 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:43:26.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:26 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:43:26.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:26 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:43:26.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:26 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:26.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:26 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi078"} : dispatch 2024-09-17T09:43:26.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:26 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi078"}]': finished 2024-09-17T09:43:26.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:26 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi136"} : dispatch 2024-09-17T09:43:26.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:26 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi136"}]': finished 2024-09-17T09:43:26.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:26 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-17T09:43:26.815 INFO:teuthology.orchestra.run.smithi078.stdout: 5 179066 10.87 MB/sec execute 261 sec latency 57.527 ms 2024-09-17T09:43:27.506 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:27 smithi078 ceph-mon[63787]: Upgrade: Setting container_image for all crash 2024-09-17T09:43:27.507 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:27 smithi078 ceph-mon[63787]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-17T09:43:27.507 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:27 smithi078 ceph-mon[63787]: Upgrade: osd.0 is safe to restart 2024-09-17T09:43:27.507 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:27 smithi078 ceph-mon[63787]: pgmap v50: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.4 MiB/s wr, 435 op/s 2024-09-17T09:43:27.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:27 smithi136 ceph-mon[49560]: Upgrade: Setting container_image for all crash 2024-09-17T09:43:27.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:27 smithi136 ceph-mon[49560]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-17T09:43:27.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:27 smithi136 ceph-mon[49560]: Upgrade: osd.0 is safe to restart 2024-09-17T09:43:27.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:27 smithi136 ceph-mon[49560]: pgmap v50: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.4 MiB/s wr, 435 op/s 2024-09-17T09:43:27.815 INFO:teuthology.orchestra.run.smithi078.stdout: 5 179496 10.85 MB/sec execute 262 sec latency 64.923 ms 2024-09-17T09:43:28.816 INFO:teuthology.orchestra.run.smithi078.stdout: 5 179958 10.87 MB/sec execute 263 sec latency 41.601 ms 2024-09-17T09:43:29.546 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:29 smithi078 ceph-mon[63787]: pgmap v51: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.4 MiB/s wr, 435 op/s 2024-09-17T09:43:29.547 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:29 smithi078 ceph-mon[63787]: Upgrade: Updating osd.0 2024-09-17T09:43:29.547 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:29 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:29.547 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:29 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-17T09:43:29.547 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:29 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:43:29.547 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:29 smithi078 ceph-mon[63787]: Deploying daemon osd.0 on smithi078 2024-09-17T09:43:29.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:29 smithi136 ceph-mon[49560]: pgmap v51: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.4 MiB/s wr, 435 op/s 2024-09-17T09:43:29.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:29 smithi136 ceph-mon[49560]: Upgrade: Updating osd.0 2024-09-17T09:43:29.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:29 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:29.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:29 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-17T09:43:29.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:29 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:43:29.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:29 smithi136 ceph-mon[49560]: Deploying daemon osd.0 on smithi078 2024-09-17T09:43:29.815 INFO:teuthology.orchestra.run.smithi078.stdout: 5 180517 10.91 MB/sec execute 264 sec latency 31.745 ms 2024-09-17T09:43:30.816 INFO:teuthology.orchestra.run.smithi078.stdout: 5 180877 10.92 MB/sec execute 265 sec latency 47.841 ms 2024-09-17T09:43:31.565 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:31 smithi078 ceph-mon[63787]: pgmap v52: 97 pgs: 97 active+clean; 340 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 696 op/s 2024-09-17T09:43:31.816 INFO:teuthology.orchestra.run.smithi078.stdout: 5 181200 10.91 MB/sec execute 266 sec latency 43.450 ms 2024-09-17T09:43:31.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:31 smithi136 ceph-mon[49560]: pgmap v52: 97 pgs: 97 active+clean; 340 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 696 op/s 2024-09-17T09:43:32.816 INFO:teuthology.orchestra.run.smithi078.stdout: 5 181615 10.90 MB/sec execute 267 sec latency 41.605 ms 2024-09-17T09:43:33.586 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:33 smithi078 ceph-mon[63787]: pgmap v53: 97 pgs: 97 active+clean; 340 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.2 MiB/s wr, 505 op/s 2024-09-17T09:43:33.816 INFO:teuthology.orchestra.run.smithi078.stdout: 5 182022 10.90 MB/sec execute 268 sec latency 57.266 ms 2024-09-17T09:43:33.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:33 smithi136 ceph-mon[49560]: pgmap v53: 97 pgs: 97 active+clean; 340 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.2 MiB/s wr, 505 op/s 2024-09-17T09:43:34.816 INFO:teuthology.orchestra.run.smithi078.stdout: 5 182478 10.89 MB/sec execute 269 sec latency 98.979 ms 2024-09-17T09:43:35.094 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:34 smithi078 systemd[1]: Stopping Ceph osd.0 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:43:35.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:35 smithi078 ceph-mon[63787]: pgmap v54: 97 pgs: 97 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.9 MiB/s wr, 752 op/s 2024-09-17T09:43:35.816 INFO:teuthology.orchestra.run.smithi078.stdout: 5 183168 10.91 MB/sec execute 270 sec latency 74.722 ms 2024-09-17T09:43:35.912 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:35 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0[34191]: 2024-09-17T09:43:35.763+0000 7f5112927700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:43:35.912 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:35 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0[34191]: 2024-09-17T09:43:35.763+0000 7f5112927700 -1 osd.0 73 *** Got signal Terminated *** 2024-09-17T09:43:35.912 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:35 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0[34191]: 2024-09-17T09:43:35.763+0000 7f5112927700 -1 osd.0 73 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:43:35.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:35 smithi136 ceph-mon[49560]: pgmap v54: 97 pgs: 97 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 7.9 MiB/s wr, 752 op/s 2024-09-17T09:43:36.816 INFO:teuthology.orchestra.run.smithi078.stdout: 5 183405 10.89 MB/sec execute 271 sec latency 287.082 ms 2024-09-17T09:43:36.958 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:36 smithi078 ceph-mon[63787]: osd.0 marked itself down and dead 2024-09-17T09:43:36.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:36 smithi136 ceph-mon[49560]: osd.0 marked itself down and dead 2024-09-17T09:43:37.346 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:37 smithi078 podman[68116]: 2024-09-17 09:43:37.140179908 +0000 UTC m=+2.008615663 container died b4ca42f8477d743d61d63228d6d8a7d684ea05f5a546261193c5882b4c2b9a02 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, release=754, architecture=x86_64, maintainer=Guillaume Abrioux , name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, GIT_CLEAN=True, vcs-type=git, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8) 2024-09-17T09:43:37.736 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:37 smithi078 ceph-mon[63787]: pgmap v55: 97 pgs: 97 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 506 op/s 2024-09-17T09:43:37.737 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:37 smithi078 ceph-mon[63787]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:43:37.737 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:37 smithi078 ceph-mon[63787]: osdmap e74: 8 total, 7 up, 8 in 2024-09-17T09:43:37.816 INFO:teuthology.orchestra.run.smithi078.stdout: 5 183532 10.87 MB/sec execute 272 sec latency 989.919 ms 2024-09-17T09:43:37.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:37 smithi136 ceph-mon[49560]: pgmap v55: 97 pgs: 97 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.1 MiB/s wr, 506 op/s 2024-09-17T09:43:37.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:37 smithi136 ceph-mon[49560]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:43:37.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:37 smithi136 ceph-mon[49560]: osdmap e74: 8 total, 7 up, 8 in 2024-09-17T09:43:38.238 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:38 smithi078 podman[68116]: 2024-09-17 09:43:38.203692525 +0000 UTC m=+3.072128279 container cleanup b4ca42f8477d743d61d63228d6d8a7d684ea05f5a546261193c5882b4c2b9a02 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, architecture=x86_64, release=754, io.k8s.display-name=CentOS Stream 8, version=8, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , name=centos-stream, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, ceph=True, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-17T09:43:38.239 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:38 smithi078 bash[68116]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0 2024-09-17T09:43:38.750 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:38 smithi078 ceph-mon[63787]: osdmap e75: 8 total, 7 up, 8 in 2024-09-17T09:43:38.750 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:38.750 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:43:38.750 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:38 smithi078 podman[68130]: 2024-09-17 09:43:38.692622991 +0000 UTC m=+1.547893961 container remove b4ca42f8477d743d61d63228d6d8a7d684ea05f5a546261193c5882b4c2b9a02 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, RELEASE=HEAD, architecture=x86_64, vcs-type=git, io.buildah.version=1.19.8, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-17T09:43:38.816 INFO:teuthology.orchestra.run.smithi078.stdout: 5 184030 10.90 MB/sec execute 273 sec latency 43.424 ms 2024-09-17T09:43:38.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:38 smithi136 ceph-mon[49560]: osdmap e75: 8 total, 7 up, 8 in 2024-09-17T09:43:38.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:38.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:43:39.513 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:39 smithi078 ceph-mon[63787]: pgmap v58: 97 pgs: 13 stale+active+clean, 84 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.1 MiB/s wr, 369 op/s 2024-09-17T09:43:39.514 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:39 smithi078 podman[68217]: 2024-09-17 09:43:39.167156438 +0000 UTC m=+0.022808139 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:43:39.514 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:39 smithi078 podman[68217]: 2024-09-17 09:43:39.429870088 +0000 UTC m=+0.285521778 container create 5d844b131259af1edd880c7865ac21bf2e01e5c186944f7d3545dc336e29909e (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS) 2024-09-17T09:43:39.816 INFO:teuthology.orchestra.run.smithi078.stdout: 5 184317 10.88 MB/sec execute 274 sec latency 54.407 ms 2024-09-17T09:43:39.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:39 smithi136 ceph-mon[49560]: pgmap v58: 97 pgs: 13 stale+active+clean, 84 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.1 MiB/s wr, 369 op/s 2024-09-17T09:43:40.274 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:40 smithi078 podman[68217]: 2024-09-17 09:43:40.143464009 +0000 UTC m=+0.999115702 container init 5d844b131259af1edd880c7865ac21bf2e01e5c186944f7d3545dc336e29909e (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T09:43:40.275 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:40 smithi078 podman[68217]: 2024-09-17 09:43:40.155674233 +0000 UTC m=+1.011325928 container start 5d844b131259af1edd880c7865ac21bf2e01e5c186944f7d3545dc336e29909e (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0-deactivate, ceph=True, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T09:43:40.526 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:40 smithi078 podman[68217]: 2024-09-17 09:43:40.4012322 +0000 UTC m=+1.256883888 container attach 5d844b131259af1edd880c7865ac21bf2e01e5c186944f7d3545dc336e29909e (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, CEPH_POINT_RELEASE=) 2024-09-17T09:43:40.526 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:40 smithi078 podman[68217]: 2024-09-17 09:43:40.487510912 +0000 UTC m=+1.343162604 container died 5d844b131259af1edd880c7865ac21bf2e01e5c186944f7d3545dc336e29909e (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0-deactivate, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-17T09:43:40.579 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:40 smithi078 ceph-mon[63787]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.9 KiB/s rd, 7.5 MiB/s wr, 740 op/s; 217/1626 objects degraded (13.346%) 2024-09-17T09:43:40.579 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:40 smithi078 ceph-mon[63787]: Health check failed: Degraded data redundancy: 217/1626 objects degraded (13.346%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T09:43:40.816 INFO:teuthology.orchestra.run.smithi078.stdout: 5 184635 10.88 MB/sec execute 275 sec latency 57.896 ms 2024-09-17T09:43:40.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:40 smithi136 ceph-mon[49560]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.9 KiB/s rd, 7.5 MiB/s wr, 740 op/s; 217/1626 objects degraded (13.346%) 2024-09-17T09:43:40.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:40 smithi136 ceph-mon[49560]: Health check failed: Degraded data redundancy: 217/1626 objects degraded (13.346%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T09:43:41.816 INFO:teuthology.orchestra.run.smithi078.stdout: 5 184993 10.87 MB/sec execute 276 sec latency 63.408 ms 2024-09-17T09:43:42.040 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:41 smithi078 podman[68241]: 2024-09-17 09:43:41.999490363 +0000 UTC m=+1.503652356 container remove 5d844b131259af1edd880c7865ac21bf2e01e5c186944f7d3545dc336e29909e (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS) 2024-09-17T09:43:42.294 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:42 smithi078 systemd[1]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.0.service: Deactivated successfully. 2024-09-17T09:43:42.294 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:42 smithi078 systemd[1]: Stopped Ceph osd.0 for 35c0cd22-74d7-11ef-bceb-c7b262605968. 2024-09-17T09:43:42.295 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:42 smithi078 systemd[1]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.0.service: Consumed 3min 2.278s CPU time. 2024-09-17T09:43:42.545 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:42 smithi078 systemd[1]: Starting Ceph osd.0 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:43:42.799 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:42 smithi078 podman[68336]: 2024-09-17 09:43:42.613432957 +0000 UTC m=+0.024840760 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:43:42.816 INFO:teuthology.orchestra.run.smithi078.stdout: 5 185551 10.88 MB/sec execute 277 sec latency 45.198 ms 2024-09-17T09:43:43.050 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:42 smithi078 podman[68336]: 2024-09-17 09:43:42.812818884 +0000 UTC m=+0.224226676 container create 30e2f6b48a68e9835b2a0d1ef3f4969afac0207ef1a3845695fc05cc46c13707 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T09:43:43.307 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:43 smithi078 podman[68336]: 2024-09-17 09:43:43.289909552 +0000 UTC m=+0.701317349 container init 30e2f6b48a68e9835b2a0d1ef3f4969afac0207ef1a3845695fc05cc46c13707 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0-activate, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T09:43:43.308 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:43 smithi078 podman[68336]: 2024-09-17 09:43:43.295004794 +0000 UTC m=+0.706412586 container start 30e2f6b48a68e9835b2a0d1ef3f4969afac0207ef1a3845695fc05cc46c13707 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-17T09:43:43.558 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:43 smithi078 ceph-mon[63787]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.4 MiB/s wr, 371 op/s; 217/1626 objects degraded (13.346%) 2024-09-17T09:43:43.559 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:43 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0-activate[68349]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:43:43.559 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:43 smithi078 podman[68336]: 2024-09-17 09:43:43.477285501 +0000 UTC m=+0.888693294 container attach 30e2f6b48a68e9835b2a0d1ef3f4969afac0207ef1a3845695fc05cc46c13707 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0-activate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, io.buildah.version=1.37.2) 2024-09-17T09:43:43.559 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:43 smithi078 bash[68336]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:43:43.559 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:43 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0-activate[68349]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:43:43.559 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:43 smithi078 bash[68336]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:43:43.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:43 smithi136 ceph-mon[49560]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.4 MiB/s wr, 371 op/s; 217/1626 objects degraded (13.346%) 2024-09-17T09:43:43.816 INFO:teuthology.orchestra.run.smithi078.stdout: 5 185923 10.86 MB/sec execute 278 sec latency 68.581 ms 2024-09-17T09:43:44.634 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-17T09:43:44.816 INFO:teuthology.orchestra.run.smithi078.stdout: 5 186251 10.85 MB/sec execute 279 sec latency 44.662 ms 2024-09-17T09:43:45.093 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:45 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0-activate[68349]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-17T09:43:45.094 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:45 smithi078 bash[68336]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-17T09:43:45.094 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:45 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0-activate[68349]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-17T09:43:45.094 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:45 smithi078 bash[68336]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-17T09:43:45.600 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:45 smithi078 ceph-mon[63787]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.2 MiB/s wr, 718 op/s; 224/1719 objects degraded (13.031%) 2024-09-17T09:43:45.600 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:45 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0-activate[68349]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-17T09:43:45.601 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:45 smithi078 bash[68336]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-17T09:43:45.601 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:45 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0-activate[68349]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-17T09:43:45.601 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:45 smithi078 bash[68336]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-17T09:43:45.601 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:45 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0-activate[68349]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-17T09:43:45.601 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:45 smithi078 bash[68336]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-17T09:43:45.601 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:45 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0-activate[68349]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-17T09:43:45.601 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:45 smithi078 bash[68336]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-17T09:43:45.601 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:45 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0-activate[68349]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-17T09:43:45.601 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:45 smithi078 bash[68336]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-17T09:43:45.601 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:45 smithi078 podman[68336]: 2024-09-17 09:43:45.47339615 +0000 UTC m=+2.884803946 container died 30e2f6b48a68e9835b2a0d1ef3f4969afac0207ef1a3845695fc05cc46c13707 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T09:43:45.777 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:43:45.777 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (4m) 28s ago 13m 19.2M - 0.27.0 11f11916f8cd 97db2373ce92 2024-09-17T09:43:45.777 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (45s) 28s ago 13m 6899k - 19.3.0-4963-ge646f729 f271290a0950 da528c5ca445 2024-09-17T09:43:45.778 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi136 smithi136 running (39s) 35s ago 12m 6899k - 19.3.0-4963-ge646f729 f271290a0950 909f405348cc 2024-09-17T09:43:45.778 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (4m) 28s ago 13m 52.4M - dad864ee21e9 db74b0f8e0a7 2024-09-17T09:43:45.778 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.jdrsup smithi078 running (7m) 28s ago 7m 38.0M - 17.2.0 e1d6a67b021e 8f834e15ba88 2024-09-17T09:43:45.778 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi136.smslff smithi136 running (7m) 35s ago 7m 15.8M - 17.2.0 e1d6a67b021e 0c20e2b328d3 2024-09-17T09:43:45.778 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rsqofi smithi078 *:8443,9283,8765 running (3m) 28s ago 14m 509M - 19.3.0-4963-ge646f729 f271290a0950 7dda3108c9a0 2024-09-17T09:43:45.778 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi136.hicmlm smithi136 *:8443,9283,8765 running (2m) 35s ago 12m 453M - 19.3.0-4963-ge646f729 f271290a0950 fc518b58e07a 2024-09-17T09:43:45.778 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (2m) 28s ago 14m 45.1M 2048M 19.3.0-4963-ge646f729 f271290a0950 3ead101d3db1 2024-09-17T09:43:45.778 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi136 smithi136 running (97s) 35s ago 12m 36.0M 2048M 19.3.0-4963-ge646f729 f271290a0950 4fce88842b79 2024-09-17T09:43:45.778 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.tcwbvi smithi078 *:2049 running (6m) 28s ago 6m 280M - 4.0 e1d6a67b021e 30c87cb46555 2024-09-17T09:43:45.778 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi136.regjvc smithi136 *:2049 running (6m) 35s ago 6m 87.7M - 4.0 e1d6a67b021e b88e4dc24376 2024-09-17T09:43:45.778 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (3m) 28s ago 12m 10.4M - 1.7.0 72c9c2088986 52dad3adeeaa 2024-09-17T09:43:45.778 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi136 smithi136 *:9100 running (3m) 35s ago 12m 11.9M - 1.7.0 72c9c2088986 f2bc5856e35a 2024-09-17T09:43:45.778 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (11m) 28s ago 11m 330M 1936M 17.2.0 e1d6a67b021e b4ca42f8477d 2024-09-17T09:43:45.778 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (11m) 28s ago 11m 523M 1936M 17.2.0 e1d6a67b021e 351873014de4 2024-09-17T09:43:45.779 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (10m) 28s ago 10m 372M 1936M 17.2.0 e1d6a67b021e 07eb71e4fa57 2024-09-17T09:43:45.779 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (10m) 28s ago 10m 518M 1936M 17.2.0 e1d6a67b021e 1b3a8be89881 2024-09-17T09:43:45.779 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi136 running (9m) 35s ago 9m 557M 2704M 17.2.0 e1d6a67b021e a0c5b4de2414 2024-09-17T09:43:45.779 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi136 running (9m) 35s ago 9m 383M 2704M 17.2.0 e1d6a67b021e 30a3049955f9 2024-09-17T09:43:45.779 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi136 running (9m) 35s ago 9m 572M 2704M 17.2.0 e1d6a67b021e 1698e1567c33 2024-09-17T09:43:45.779 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi136 running (8m) 35s ago 8m 453M 2704M 17.2.0 e1d6a67b021e 5d3b08cecd78 2024-09-17T09:43:45.779 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (2m) 28s ago 12m 49.3M - 2.51.0 1d3b7f56885b 6e5e8c721eb0 2024-09-17T09:43:45.816 INFO:teuthology.orchestra.run.smithi078.stdout: 5 186710 10.86 MB/sec execute 280 sec latency 40.829 ms 2024-09-17T09:43:45.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:45 smithi136 ceph-mon[49560]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.2 MiB/s wr, 718 op/s; 224/1719 objects degraded (13.031%) 2024-09-17T09:43:46.434 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:43:46.434 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-17T09:43:46.434 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:43:46.435 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:43:46.435 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-17T09:43:46.435 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:43:46.435 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:43:46.435 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-17T09:43:46.435 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7 2024-09-17T09:43:46.435 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:43:46.435 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-17T09:43:46.435 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:43:46.435 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:43:46.435 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-17T09:43:46.435 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-17T09:43:46.436 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 4 2024-09-17T09:43:46.436 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-17T09:43:46.436 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:43:46.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:46 smithi078 ceph-mon[63787]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:43:46.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:46 smithi078 ceph-mon[63787]: from='client.34204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:43:46.618 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:46 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/366558482' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:43:46.816 INFO:teuthology.orchestra.run.smithi078.stdout: 5 187035 10.86 MB/sec execute 281 sec latency 48.356 ms 2024-09-17T09:43:46.870 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:46 smithi078 podman[68581]: 2024-09-17 09:43:46.657023027 +0000 UTC m=+1.176968591 container remove 30e2f6b48a68e9835b2a0d1ef3f4969afac0207ef1a3845695fc05cc46c13707 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0-activate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:43:46.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:46 smithi136 ceph-mon[49560]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:43:46.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:46 smithi136 ceph-mon[49560]: from='client.34204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:43:46.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:46 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/366558482' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:43:46.984 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:43:46.984 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407", 2024-09-17T09:43:46.984 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-17T09:43:46.984 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T09:43:46.984 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-17T09:43:46.984 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-09-17T09:43:46.984 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-17T09:43:46.984 INFO:teuthology.orchestra.run.smithi078.stdout: "mon" 2024-09-17T09:43:46.984 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-17T09:43:46.984 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "6/23 daemons upgraded", 2024-09-17T09:43:46.985 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T09:43:46.985 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-17T09:43:46.985 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:43:47.122 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:46 smithi078 podman[68680]: 2024-09-17 09:43:46.82899245 +0000 UTC m=+0.025430283 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:43:47.122 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:47 smithi078 podman[68680]: 2024-09-17 09:43:47.002943085 +0000 UTC m=+0.199380927 container create 1ea81cc809b834b415f1f06ca195a8ec0906e0a1e7c5565938ad2d3e4765a8a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-17T09:43:47.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:47 smithi078 ceph-mon[63787]: from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:43:47.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:47 smithi078 ceph-mon[63787]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.8 MiB/s wr, 580 op/s; 224/1719 objects degraded (13.031%) 2024-09-17T09:43:47.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:47 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 224/1719 objects degraded (13.031%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T09:43:47.624 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:47 smithi078 podman[68680]: 2024-09-17 09:43:47.461279761 +0000 UTC m=+0.657717605 container init 1ea81cc809b834b415f1f06ca195a8ec0906e0a1e7c5565938ad2d3e4765a8a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:43:47.624 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:47 smithi078 podman[68680]: 2024-09-17 09:43:47.472743008 +0000 UTC m=+0.669180844 container start 1ea81cc809b834b415f1f06ca195a8ec0906e0a1e7c5565938ad2d3e4765a8a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-17T09:43:47.624 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:47 smithi078 ceph-osd[68720]: -- 172.21.15.78:0/2427157469 <== mon.0 v2:172.21.15.78:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d964ffc1e0 con 0x55d964209c00 2024-09-17T09:43:47.700 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 224/1719 objects degraded (13.031%), 29 pgs degraded 2024-09-17T09:43:47.700 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-17T09:43:47.700 INFO:teuthology.orchestra.run.smithi078.stdout: osd.0 (root=default,host=smithi078) is down 2024-09-17T09:43:47.700 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 224/1719 objects degraded (13.031%), 29 pgs degraded 2024-09-17T09:43:47.700 INFO:teuthology.orchestra.run.smithi078.stdout: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-09-17T09:43:47.701 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.0 is active+undersized+degraded, acting [7,1] 2024-09-17T09:43:47.701 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1 is active+undersized+degraded, acting [2,3] 2024-09-17T09:43:47.701 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.4 is active+undersized+degraded, acting [1,7] 2024-09-17T09:43:47.701 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.5 is active+undersized+degraded, acting [7,4] 2024-09-17T09:43:47.701 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.8 is active+undersized+degraded, acting [7,1] 2024-09-17T09:43:47.701 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.c is active+undersized+degraded, acting [2,5] 2024-09-17T09:43:47.701 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.f is active+undersized+degraded, acting [4,7] 2024-09-17T09:43:47.701 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.10 is active+undersized+degraded, acting [2,1] 2024-09-17T09:43:47.701 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.13 is active+undersized+degraded, acting [4,3] 2024-09-17T09:43:47.701 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.15 is active+undersized+degraded, acting [1,5] 2024-09-17T09:43:47.701 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.19 is active+undersized+degraded, acting [3,6] 2024-09-17T09:43:47.701 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1d is active+undersized+degraded, acting [7,6] 2024-09-17T09:43:47.701 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1e is active+undersized+degraded, acting [3,5] 2024-09-17T09:43:47.702 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1f is active+undersized+degraded, acting [7,4] 2024-09-17T09:43:47.702 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-09-17T09:43:47.702 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-09-17T09:43:47.702 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-09-17T09:43:47.702 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-09-17T09:43:47.702 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.b is active+undersized+degraded, acting [3,4] 2024-09-17T09:43:47.702 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.f is active+undersized+degraded, acting [7,4] 2024-09-17T09:43:47.702 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-09-17T09:43:47.702 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-09-17T09:43:47.702 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-09-17T09:43:47.702 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-09-17T09:43:47.702 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-09-17T09:43:47.702 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-09-17T09:43:47.702 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-09-17T09:43:47.703 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.19 is active+undersized+degraded, acting [3,2] 2024-09-17T09:43:47.816 INFO:teuthology.orchestra.run.smithi078.stdout: 5 187718 10.90 MB/sec execute 282 sec latency 29.228 ms 2024-09-17T09:43:47.876 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:47 smithi136 ceph-mon[49560]: from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:43:47.877 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:47 smithi136 ceph-mon[49560]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.8 MiB/s wr, 580 op/s; 224/1719 objects degraded (13.031%) 2024-09-17T09:43:47.877 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:47 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 224/1719 objects degraded (13.031%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T09:43:47.877 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:47 smithi078 bash[68680]: 1ea81cc809b834b415f1f06ca195a8ec0906e0a1e7c5565938ad2d3e4765a8a2 2024-09-17T09:43:47.877 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:47 smithi078 systemd[1]: Started Ceph osd.0 for 35c0cd22-74d7-11ef-bceb-c7b262605968. 2024-09-17T09:43:48.588 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:48 smithi078 ceph-mon[63787]: from='client.34214 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:43:48.588 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:48 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/735937163' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:43:48.588 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:48 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:48.589 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:48 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:48.589 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:48 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:43:48.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:48 smithi136 ceph-mon[49560]: from='client.34214 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:43:48.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:48 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/735937163' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:43:48.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:48 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:48.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:48 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:48.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:48 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:43:48.817 INFO:teuthology.orchestra.run.smithi078.stdout: 5 188269 10.93 MB/sec execute 283 sec latency 25.277 ms 2024-09-17T09:43:49.387 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:49 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0[68716]: 2024-09-17T09:43:49.370+0000 7f9350f65640 -1 Falling back to public interface 2024-09-17T09:43:49.637 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:49 smithi078 ceph-mon[63787]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.3 MiB/s wr, 527 op/s; 224/1719 objects degraded (13.031%) 2024-09-17T09:43:49.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:49 smithi136 ceph-mon[49560]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 346 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.3 MiB/s wr, 527 op/s; 224/1719 objects degraded (13.031%) 2024-09-17T09:43:49.817 INFO:teuthology.orchestra.run.smithi078.stdout: 5 188798 10.92 MB/sec execute 284 sec latency 30.550 ms 2024-09-17T09:43:50.817 INFO:teuthology.orchestra.run.smithi078.stdout: 5 189327 10.93 MB/sec execute 285 sec latency 32.802 ms 2024-09-17T09:43:51.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:51 smithi078 ceph-mon[63787]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 347 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.6 MiB/s wr, 746 op/s; 223/1743 objects degraded (12.794%) 2024-09-17T09:43:51.817 INFO:teuthology.orchestra.run.smithi078.stdout: 5 189893 10.93 MB/sec execute 286 sec latency 89.485 ms 2024-09-17T09:43:51.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:51 smithi136 ceph-mon[49560]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 347 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.6 MiB/s wr, 746 op/s; 223/1743 objects degraded (12.794%) 2024-09-17T09:43:52.655 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:52 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 223/1743 objects degraded (12.794%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T09:43:52.817 INFO:teuthology.orchestra.run.smithi078.stdout: 5 190467 10.94 MB/sec execute 287 sec latency 178.827 ms 2024-09-17T09:43:52.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:52 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 223/1743 objects degraded (12.794%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T09:43:53.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:53 smithi078 ceph-mon[63787]: pgmap v65: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 347 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.3 MiB/s wr, 499 op/s; 223/1743 objects degraded (12.794%) 2024-09-17T09:43:53.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:53 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:53.603 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:53 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:43:53.817 INFO:teuthology.orchestra.run.smithi078.stdout: 5 190809 10.96 MB/sec execute 288 sec latency 54.101 ms 2024-09-17T09:43:53.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:53 smithi136 ceph-mon[49560]: pgmap v65: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 347 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.3 MiB/s wr, 499 op/s; 223/1743 objects degraded (12.794%) 2024-09-17T09:43:53.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:53 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:53.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:53 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:43:54.817 INFO:teuthology.orchestra.run.smithi078.stdout: 5 191127 10.96 MB/sec execute 289 sec latency 62.835 ms 2024-09-17T09:43:55.661 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:55 smithi078 ceph-mon[63787]: pgmap v66: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 823 op/s; 206/1635 objects degraded (12.599%) 2024-09-17T09:43:55.817 INFO:teuthology.orchestra.run.smithi078.stdout: 5 191377 10.94 MB/sec execute 290 sec latency 46.436 ms 2024-09-17T09:43:55.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:55 smithi136 ceph-mon[49560]: pgmap v66: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 823 op/s; 206/1635 objects degraded (12.599%) 2024-09-17T09:43:56.817 INFO:teuthology.orchestra.run.smithi078.stdout: 5 191628 10.94 MB/sec execute 291 sec latency 66.046 ms 2024-09-17T09:43:57.475 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:57 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0[68716]: 2024-09-17T09:43:57.454+0000 7f9350f65640 -1 osd.0 0 read_superblock omap replica is missing. 2024-09-17T09:43:57.725 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:57 smithi078 ceph-mon[63787]: pgmap v67: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.0 MiB/s wr, 592 op/s; 206/1635 objects degraded (12.599%) 2024-09-17T09:43:57.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:57 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:57.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:57 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:57.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:57 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 206/1635 objects degraded (12.599%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T09:43:57.817 INFO:teuthology.orchestra.run.smithi078.stdout: 5 191855 10.92 MB/sec execute 292 sec latency 46.422 ms 2024-09-17T09:43:57.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:57 smithi136 ceph-mon[49560]: pgmap v67: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.0 MiB/s wr, 592 op/s; 206/1635 objects degraded (12.599%) 2024-09-17T09:43:57.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:57 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:57.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:57 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:57.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:57 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 206/1635 objects degraded (12.599%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T09:43:58.817 INFO:teuthology.orchestra.run.smithi078.stdout: 5 192065 10.89 MB/sec execute 293 sec latency 59.585 ms 2024-09-17T09:43:59.034 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:58 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0[68716]: 2024-09-17T09:43:58.778+0000 7f9350f65640 -1 osd.0 73 log_to_monitors true 2024-09-17T09:43:59.291 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:59 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:59.292 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:59 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:59.292 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:59 smithi078 ceph-mon[63787]: pgmap v68: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.0 MiB/s wr, 592 op/s; 206/1635 objects degraded (12.599%) 2024-09-17T09:43:59.292 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:43:59 smithi078 ceph-mon[63787]: from='osd.0 [v2:172.21.15.78:6802/198833623,v1:172.21.15.78:6803/198833623]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-17T09:43:59.546 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:43:59 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0[68716]: 2024-09-17T09:43:59.378+0000 7f9348d0f640 -1 osd.0 73 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T09:43:59.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:59 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:59.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:59 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:43:59.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:59 smithi136 ceph-mon[49560]: pgmap v68: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 344 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.0 MiB/s wr, 592 op/s; 206/1635 objects degraded (12.599%) 2024-09-17T09:43:59.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:43:59 smithi136 ceph-mon[49560]: from='osd.0 [v2:172.21.15.78:6802/198833623,v1:172.21.15.78:6803/198833623]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-17T09:43:59.817 INFO:teuthology.orchestra.run.smithi078.stdout: 5 192364 10.88 MB/sec execute 294 sec latency 52.609 ms 2024-09-17T09:44:00.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:00 smithi078 ceph-mon[63787]: from='osd.0 [v2:172.21.15.78:6802/198833623,v1:172.21.15.78:6803/198833623]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-17T09:44:00.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:00 smithi078 ceph-mon[63787]: osdmap e76: 8 total, 7 up, 8 in 2024-09-17T09:44:00.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:00 smithi078 ceph-mon[63787]: from='osd.0 [v2:172.21.15.78:6802/198833623,v1:172.21.15.78:6803/198833623]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]} : dispatch 2024-09-17T09:44:00.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:00 smithi136 ceph-mon[49560]: from='osd.0 [v2:172.21.15.78:6802/198833623,v1:172.21.15.78:6803/198833623]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-17T09:44:00.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:00 smithi136 ceph-mon[49560]: osdmap e76: 8 total, 7 up, 8 in 2024-09-17T09:44:00.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:00 smithi136 ceph-mon[49560]: from='osd.0 [v2:172.21.15.78:6802/198833623,v1:172.21.15.78:6803/198833623]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]} : dispatch 2024-09-17T09:44:00.817 INFO:teuthology.orchestra.run.smithi078.stdout: 5 192542 10.87 MB/sec execute 295 sec latency 519.561 ms 2024-09-17T09:44:01.580 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:01 smithi078 ceph-mon[63787]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:44:01.581 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:01 smithi078 ceph-mon[63787]: osd.0 [v2:172.21.15.78:6802/198833623,v1:172.21.15.78:6803/198833623] boot 2024-09-17T09:44:01.581 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:01 smithi078 ceph-mon[63787]: osdmap e77: 8 total, 8 up, 8 in 2024-09-17T09:44:01.581 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:01 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-17T09:44:01.581 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:01 smithi078 ceph-mon[63787]: pgmap v71: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.5 MiB/s wr, 723 op/s; 220/1698 objects degraded (12.956%) 2024-09-17T09:44:01.581 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:01 smithi078 ceph-mon[63787]: osdmap e78: 8 total, 8 up, 8 in 2024-09-17T09:44:01.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:01 smithi136 ceph-mon[49560]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:44:01.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:01 smithi136 ceph-mon[49560]: osd.0 [v2:172.21.15.78:6802/198833623,v1:172.21.15.78:6803/198833623] boot 2024-09-17T09:44:01.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:01 smithi136 ceph-mon[49560]: osdmap e77: 8 total, 8 up, 8 in 2024-09-17T09:44:01.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:01 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-17T09:44:01.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:01 smithi136 ceph-mon[49560]: pgmap v71: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.5 MiB/s wr, 723 op/s; 220/1698 objects degraded (12.956%) 2024-09-17T09:44:01.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:01 smithi136 ceph-mon[49560]: osdmap e78: 8 total, 8 up, 8 in 2024-09-17T09:44:01.817 INFO:teuthology.orchestra.run.smithi078.stdout: 5 192633 10.83 MB/sec execute 296 sec latency 1512.768 ms 2024-09-17T09:44:02.817 INFO:teuthology.orchestra.run.smithi078.stdout: 5 192658 10.80 MB/sec execute 297 sec latency 2512.795 ms 2024-09-17T09:44:03.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:03 smithi078 ceph-mon[63787]: osdmap e79: 8 total, 8 up, 8 in 2024-09-17T09:44:03.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:03 smithi078 ceph-mon[63787]: pgmap v74: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.5 MiB/s wr, 473 op/s; 220/1698 objects degraded (12.956%) 2024-09-17T09:44:03.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:03 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 220/1698 objects degraded (12.956%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T09:44:03.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:03 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:03.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:03 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:03.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:03 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:44:03.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:03 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:44:03.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:03 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:03.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:03 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:03.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:03 smithi136 ceph-mon[49560]: osdmap e79: 8 total, 8 up, 8 in 2024-09-17T09:44:03.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:03 smithi136 ceph-mon[49560]: pgmap v74: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.5 MiB/s wr, 473 op/s; 220/1698 objects degraded (12.956%) 2024-09-17T09:44:03.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:03 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 220/1698 objects degraded (12.956%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T09:44:03.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:03 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:03.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:03 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:03.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:03 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:44:03.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:03 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:44:03.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:03 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:03.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:03 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:03.817 INFO:teuthology.orchestra.run.smithi078.stdout: 5 192975 10.77 MB/sec execute 298 sec latency 2573.092 ms 2024-09-17T09:44:04.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:04 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:44:04.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:04 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:44:04.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:04 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:44:04.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:04 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:44:04.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:04 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-17T09:44:04.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:04 smithi078 ceph-mon[63787]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T09:44:04.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:04 smithi078 ceph-mon[63787]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-17T09:44:04.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:04 smithi078 ceph-mon[63787]: osdmap e80: 8 total, 8 up, 8 in 2024-09-17T09:44:04.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:04 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:44:04.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:04 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:44:04.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:04 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:44:04.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:04 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:44:04.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:04 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-17T09:44:04.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:04 smithi136 ceph-mon[49560]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T09:44:04.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:04 smithi136 ceph-mon[49560]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-17T09:44:04.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:04 smithi136 ceph-mon[49560]: osdmap e80: 8 total, 8 up, 8 in 2024-09-17T09:44:04.817 INFO:teuthology.orchestra.run.smithi078.stdout: 5 193437 10.78 MB/sec execute 299 sec latency 340.005 ms 2024-09-17T09:44:05.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:05 smithi078 ceph-mon[63787]: pgmap v76: 97 pgs: 1 peering, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 330 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 2.8 MiB/s wr, 357 op/s; 301/1464 objects degraded (20.560%); 3.0 MiB/s, 5 objects/s recovering 2024-09-17T09:44:05.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:05 smithi078 ceph-mon[63787]: osdmap e81: 8 total, 8 up, 8 in 2024-09-17T09:44:05.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:05 smithi136 ceph-mon[49560]: pgmap v76: 97 pgs: 1 peering, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 330 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 2.8 MiB/s wr, 357 op/s; 301/1464 objects degraded (20.560%); 3.0 MiB/s, 5 objects/s recovering 2024-09-17T09:44:05.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:05 smithi136 ceph-mon[49560]: osdmap e81: 8 total, 8 up, 8 in 2024-09-17T09:44:05.817 INFO:teuthology.orchestra.run.smithi078.stdout: 5 193802 10.77 MB/sec execute 300 sec latency 803.899 ms 2024-09-17T09:44:06.817 INFO:teuthology.orchestra.run.smithi078.stdout: 5 194141 10.77 MB/sec execute 301 sec latency 988.242 ms 2024-09-17T09:44:07.203 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:06 smithi078 ceph-mon[63787]: osdmap e82: 8 total, 8 up, 8 in 2024-09-17T09:44:07.203 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:06 smithi078 ceph-mon[63787]: pgmap v79: 97 pgs: 1 peering, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 330 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 2.8 MiB/s wr, 357 op/s; 301/1464 objects degraded (20.560%); 3.0 MiB/s, 5 objects/s recovering 2024-09-17T09:44:07.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:06 smithi136 ceph-mon[49560]: osdmap e82: 8 total, 8 up, 8 in 2024-09-17T09:44:07.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:06 smithi136 ceph-mon[49560]: pgmap v79: 97 pgs: 1 peering, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 330 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 2.8 MiB/s wr, 357 op/s; 301/1464 objects degraded (20.560%); 3.0 MiB/s, 5 objects/s recovering 2024-09-17T09:44:07.817 INFO:teuthology.orchestra.run.smithi078.stdout: 5 194492 10.77 MB/sec execute 302 sec latency 337.324 ms 2024-09-17T09:44:07.978 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:07 smithi078 ceph-mon[63787]: osdmap e83: 8 total, 8 up, 8 in 2024-09-17T09:44:07.979 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:07 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 301/1464 objects degraded (20.560%), 11 pgs degraded (PG_DEGRADED) 2024-09-17T09:44:07.979 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:07 smithi078 ceph-mon[63787]: osdmap e84: 8 total, 8 up, 8 in 2024-09-17T09:44:07.979 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:07 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:07.979 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:07 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:44:08.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:07 smithi136 ceph-mon[49560]: osdmap e83: 8 total, 8 up, 8 in 2024-09-17T09:44:08.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:07 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 301/1464 objects degraded (20.560%), 11 pgs degraded (PG_DEGRADED) 2024-09-17T09:44:08.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:07 smithi136 ceph-mon[49560]: osdmap e84: 8 total, 8 up, 8 in 2024-09-17T09:44:08.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:07 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:08.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:07 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:44:08.817 INFO:teuthology.orchestra.run.smithi078.stdout: 5 194784 10.78 MB/sec execute 303 sec latency 612.739 ms 2024-09-17T09:44:09.530 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:09 smithi078 ceph-mon[63787]: pgmap v82: 97 pgs: 1 peering, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 330 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 301/1464 objects degraded (20.560%) 2024-09-17T09:44:09.531 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:09 smithi078 ceph-mon[63787]: osdmap e85: 8 total, 8 up, 8 in 2024-09-17T09:44:09.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:09 smithi136 ceph-mon[49560]: pgmap v82: 97 pgs: 1 peering, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 330 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 301/1464 objects degraded (20.560%) 2024-09-17T09:44:09.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:09 smithi136 ceph-mon[49560]: osdmap e85: 8 total, 8 up, 8 in 2024-09-17T09:44:09.817 INFO:teuthology.orchestra.run.smithi078.stdout: 5 195104 10.78 MB/sec execute 304 sec latency 38.182 ms 2024-09-17T09:44:10.543 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:10 smithi078 ceph-mon[63787]: osdmap e86: 8 total, 8 up, 8 in 2024-09-17T09:44:10.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:10 smithi136 ceph-mon[49560]: osdmap e86: 8 total, 8 up, 8 in 2024-09-17T09:44:10.817 INFO:teuthology.orchestra.run.smithi078.stdout: 5 195438 10.77 MB/sec execute 305 sec latency 49.477 ms 2024-09-17T09:44:11.578 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:11 smithi078 ceph-mon[63787]: pgmap v85: 97 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 91 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 6.2 MiB/s wr, 631 op/s; 121/1632 objects degraded (7.414%); 0 B/s, 51 objects/s recovering 2024-09-17T09:44:11.578 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:11 smithi078 ceph-mon[63787]: osdmap e87: 8 total, 8 up, 8 in 2024-09-17T09:44:11.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:11 smithi136 ceph-mon[49560]: pgmap v85: 97 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 91 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 6.2 MiB/s wr, 631 op/s; 121/1632 objects degraded (7.414%); 0 B/s, 51 objects/s recovering 2024-09-17T09:44:11.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:11 smithi136 ceph-mon[49560]: osdmap e87: 8 total, 8 up, 8 in 2024-09-17T09:44:11.817 INFO:teuthology.orchestra.run.smithi078.stdout: 5 195704 10.75 MB/sec execute 306 sec latency 695.488 ms 2024-09-17T09:44:12.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:12 smithi078 ceph-mon[63787]: osdmap e88: 8 total, 8 up, 8 in 2024-09-17T09:44:12.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:12 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 121/1632 objects degraded (7.414%), 5 pgs degraded (PG_DEGRADED) 2024-09-17T09:44:12.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:12 smithi078 ceph-mon[63787]: osdmap e89: 8 total, 8 up, 8 in 2024-09-17T09:44:12.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:12 smithi136 ceph-mon[49560]: osdmap e88: 8 total, 8 up, 8 in 2024-09-17T09:44:12.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:12 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 121/1632 objects degraded (7.414%), 5 pgs degraded (PG_DEGRADED) 2024-09-17T09:44:12.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:12 smithi136 ceph-mon[49560]: osdmap e89: 8 total, 8 up, 8 in 2024-09-17T09:44:12.817 INFO:teuthology.orchestra.run.smithi078.stdout: 5 196017 10.74 MB/sec execute 307 sec latency 97.206 ms 2024-09-17T09:44:13.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:13 smithi078 ceph-mon[63787]: pgmap v88: 97 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 91 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 6.2 MiB/s wr, 631 op/s; 121/1632 objects degraded (7.414%); 0 B/s, 51 objects/s recovering 2024-09-17T09:44:13.623 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:13 smithi078 ceph-mon[63787]: osdmap e90: 8 total, 8 up, 8 in 2024-09-17T09:44:13.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:13 smithi136 ceph-mon[49560]: pgmap v88: 97 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 91 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 6.2 MiB/s wr, 631 op/s; 121/1632 objects degraded (7.414%); 0 B/s, 51 objects/s recovering 2024-09-17T09:44:13.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:13 smithi136 ceph-mon[49560]: osdmap e90: 8 total, 8 up, 8 in 2024-09-17T09:44:13.817 INFO:teuthology.orchestra.run.smithi078.stdout: 5 196341 10.73 MB/sec execute 308 sec latency 60.915 ms 2024-09-17T09:44:14.817 INFO:teuthology.orchestra.run.smithi078.stdout: 5 196752 10.72 MB/sec execute 309 sec latency 39.853 ms 2024-09-17T09:44:15.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:15 smithi078 ceph-mon[63787]: pgmap v91: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 603 op/s; 34/1602 objects degraded (2.122%); 0 B/s, 28 objects/s recovering 2024-09-17T09:44:15.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:15 smithi078 ceph-mon[63787]: osdmap e91: 8 total, 8 up, 8 in 2024-09-17T09:44:15.817 INFO:teuthology.orchestra.run.smithi078.stdout: 5 197296 10.73 MB/sec execute 310 sec latency 507.816 ms 2024-09-17T09:44:15.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:15 smithi136 ceph-mon[49560]: pgmap v91: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 603 op/s; 34/1602 objects degraded (2.122%); 0 B/s, 28 objects/s recovering 2024-09-17T09:44:15.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:15 smithi136 ceph-mon[49560]: osdmap e91: 8 total, 8 up, 8 in 2024-09-17T09:44:16.667 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:16 smithi078 ceph-mon[63787]: osdmap e92: 8 total, 8 up, 8 in 2024-09-17T09:44:16.817 INFO:teuthology.orchestra.run.smithi078.stdout: 5 197735 10.72 MB/sec execute 311 sec latency 380.752 ms 2024-09-17T09:44:16.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:16 smithi136 ceph-mon[49560]: osdmap e92: 8 total, 8 up, 8 in 2024-09-17T09:44:17.673 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:17 smithi078 ceph-mon[63787]: pgmap v94: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 603 op/s; 34/1602 objects degraded (2.122%); 0 B/s, 28 objects/s recovering 2024-09-17T09:44:17.674 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:17 smithi078 ceph-mon[63787]: osdmap e93: 8 total, 8 up, 8 in 2024-09-17T09:44:17.674 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:17 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 34/1602 objects degraded (2.122%), 2 pgs degraded (PG_DEGRADED) 2024-09-17T09:44:17.817 INFO:teuthology.orchestra.run.smithi078.stdout: 5 198254 10.76 MB/sec execute 312 sec latency 38.387 ms 2024-09-17T09:44:17.876 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:17 smithi136 ceph-mon[49560]: pgmap v94: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 603 op/s; 34/1602 objects degraded (2.122%); 0 B/s, 28 objects/s recovering 2024-09-17T09:44:17.877 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:17 smithi136 ceph-mon[49560]: osdmap e93: 8 total, 8 up, 8 in 2024-09-17T09:44:17.877 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:17 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 34/1602 objects degraded (2.122%), 2 pgs degraded (PG_DEGRADED) 2024-09-17T09:44:18.307 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-17T09:44:18.691 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-17T09:44:18.818 INFO:teuthology.orchestra.run.smithi078.stdout: 5 198654 10.78 MB/sec execute 313 sec latency 38.816 ms 2024-09-17T09:44:18.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-17T09:44:19.425 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:44:19.425 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (4m) 22s ago 13m 17.6M - 0.27.0 11f11916f8cd 97db2373ce92 2024-09-17T09:44:19.425 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (79s) 22s ago 13m 6899k - 19.3.0-4963-ge646f729 f271290a0950 da528c5ca445 2024-09-17T09:44:19.425 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi136 smithi136 running (73s) 68s ago 13m 6899k - 19.3.0-4963-ge646f729 f271290a0950 909f405348cc 2024-09-17T09:44:19.425 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (4m) 22s ago 13m 52.3M - dad864ee21e9 db74b0f8e0a7 2024-09-17T09:44:19.425 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.jdrsup smithi078 running (7m) 22s ago 7m 38.1M - 17.2.0 e1d6a67b021e 8f834e15ba88 2024-09-17T09:44:19.425 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi136.smslff smithi136 running (7m) 68s ago 7m 15.8M - 17.2.0 e1d6a67b021e 0c20e2b328d3 2024-09-17T09:44:19.426 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rsqofi smithi078 *:8443,9283,8765 running (3m) 22s ago 15m 511M - 19.3.0-4963-ge646f729 f271290a0950 7dda3108c9a0 2024-09-17T09:44:19.426 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi136.hicmlm smithi136 *:8443,9283,8765 running (2m) 68s ago 13m 453M - 19.3.0-4963-ge646f729 f271290a0950 fc518b58e07a 2024-09-17T09:44:19.426 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (2m) 22s ago 15m 48.2M 2048M 19.3.0-4963-ge646f729 f271290a0950 3ead101d3db1 2024-09-17T09:44:19.426 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi136 smithi136 running (2m) 68s ago 13m 36.0M 2048M 19.3.0-4963-ge646f729 f271290a0950 4fce88842b79 2024-09-17T09:44:19.426 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.tcwbvi smithi078 *:2049 running (7m) 22s ago 7m 277M - 4.0 e1d6a67b021e 30c87cb46555 2024-09-17T09:44:19.426 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi136.regjvc smithi136 *:2049 running (7m) 68s ago 7m 87.7M - 4.0 e1d6a67b021e b88e4dc24376 2024-09-17T09:44:19.426 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (4m) 22s ago 13m 10.2M - 1.7.0 72c9c2088986 52dad3adeeaa 2024-09-17T09:44:19.426 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi136 smithi136 *:9100 running (3m) 68s ago 12m 11.9M - 1.7.0 72c9c2088986 f2bc5856e35a 2024-09-17T09:44:19.426 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (32s) 22s ago 12m 12.0M 1936M 19.3.0-4963-ge646f729 f271290a0950 1ea81cc809b8 2024-09-17T09:44:19.426 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (11m) 22s ago 11m 569M 1936M 17.2.0 e1d6a67b021e 351873014de4 2024-09-17T09:44:19.426 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (11m) 22s ago 11m 435M 1936M 17.2.0 e1d6a67b021e 07eb71e4fa57 2024-09-17T09:44:19.426 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (10m) 22s ago 10m 577M 1936M 17.2.0 e1d6a67b021e 1b3a8be89881 2024-09-17T09:44:19.426 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi136 running (10m) 68s ago 10m 557M 2704M 17.2.0 e1d6a67b021e a0c5b4de2414 2024-09-17T09:44:19.426 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi136 running (10m) 68s ago 10m 383M 2704M 17.2.0 e1d6a67b021e 30a3049955f9 2024-09-17T09:44:19.427 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi136 running (9m) 68s ago 9m 572M 2704M 17.2.0 e1d6a67b021e 1698e1567c33 2024-09-17T09:44:19.427 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi136 running (9m) 68s ago 9m 453M 2704M 17.2.0 e1d6a67b021e 5d3b08cecd78 2024-09-17T09:44:19.427 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (3m) 22s ago 12m 51.5M - 2.51.0 1d3b7f56885b 6e5e8c721eb0 2024-09-17T09:44:19.719 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:19 smithi078 ceph-mon[63787]: from='client.34226 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:44:19.720 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:19 smithi078 ceph-mon[63787]: pgmap v96: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.1 MiB/s wr, 471 op/s; 34/1602 objects degraded (2.122%); 0 B/s, 17 objects/s recovering 2024-09-17T09:44:19.720 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:19 smithi078 ceph-mon[63787]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T09:44:19.720 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:19 smithi078 ceph-mon[63787]: Upgrade: osd.1 is safe to restart 2024-09-17T09:44:19.818 INFO:teuthology.orchestra.run.smithi078.stdout: 5 198965 10.78 MB/sec execute 314 sec latency 31.466 ms 2024-09-17T09:44:19.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:19 smithi136 ceph-mon[49560]: from='client.34226 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:44:19.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:19 smithi136 ceph-mon[49560]: pgmap v96: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.1 MiB/s wr, 471 op/s; 34/1602 objects degraded (2.122%); 0 B/s, 17 objects/s recovering 2024-09-17T09:44:19.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:19 smithi136 ceph-mon[49560]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T09:44:19.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:19 smithi136 ceph-mon[49560]: Upgrade: osd.1 is safe to restart 2024-09-17T09:44:20.031 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:44:20.031 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-17T09:44:20.031 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:44:20.031 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:44:20.032 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-17T09:44:20.032 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:44:20.032 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:44:20.032 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-17T09:44:20.032 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-17T09:44:20.032 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 1 2024-09-17T09:44:20.032 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:44:20.032 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-17T09:44:20.032 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:44:20.032 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:44:20.032 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-17T09:44:20.033 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-17T09:44:20.033 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 5 2024-09-17T09:44:20.033 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-17T09:44:20.033 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:44:20.483 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:44:20.483 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407", 2024-09-17T09:44:20.484 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-17T09:44:20.484 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T09:44:20.484 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-17T09:44:20.484 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-09-17T09:44:20.484 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-17T09:44:20.484 INFO:teuthology.orchestra.run.smithi078.stdout: "mon" 2024-09-17T09:44:20.484 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-17T09:44:20.484 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "7/23 daemons upgraded", 2024-09-17T09:44:20.484 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T09:44:20.484 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-17T09:44:20.484 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:44:20.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:20 smithi078 ceph-mon[63787]: from='client.44165 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:44:20.730 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:20 smithi078 ceph-mon[63787]: from='client.44169 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:44:20.730 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:20 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/3560374138' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:44:20.818 INFO:teuthology.orchestra.run.smithi078.stdout: 5 199186 10.75 MB/sec execute 315 sec latency 55.281 ms 2024-09-17T09:44:20.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:20 smithi136 ceph-mon[49560]: from='client.44165 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:44:20.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:20 smithi136 ceph-mon[49560]: from='client.44169 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:44:20.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:20 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/3560374138' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:44:21.139 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-17T09:44:21.747 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:21 smithi078 ceph-mon[63787]: pgmap v97: 97 pgs: 97 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.4 MiB/s wr, 530 op/s; 0 B/s, 5 objects/s recovering 2024-09-17T09:44:21.748 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:21 smithi078 ceph-mon[63787]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:44:21.748 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:21 smithi078 ceph-mon[63787]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 34/1602 objects degraded (2.122%), 2 pgs degraded) 2024-09-17T09:44:21.748 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:21 smithi078 ceph-mon[63787]: Cluster is now healthy 2024-09-17T09:44:21.748 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:21 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:21.748 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:21 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-17T09:44:21.748 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:21 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:44:21.748 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:21 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/2831367558' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:44:21.818 INFO:teuthology.orchestra.run.smithi078.stdout: 5 199413 10.72 MB/sec execute 316 sec latency 60.359 ms 2024-09-17T09:44:21.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:21 smithi136 ceph-mon[49560]: pgmap v97: 97 pgs: 97 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.4 MiB/s wr, 530 op/s; 0 B/s, 5 objects/s recovering 2024-09-17T09:44:21.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:21 smithi136 ceph-mon[49560]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:44:21.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:21 smithi136 ceph-mon[49560]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 34/1602 objects degraded (2.122%), 2 pgs degraded) 2024-09-17T09:44:21.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:21 smithi136 ceph-mon[49560]: Cluster is now healthy 2024-09-17T09:44:21.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:21 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:21.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:21 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-17T09:44:21.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:21 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:44:21.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:21 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/2831367558' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:44:22.767 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:22 smithi078 ceph-mon[63787]: Upgrade: Updating osd.1 2024-09-17T09:44:22.767 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:22 smithi078 ceph-mon[63787]: Deploying daemon osd.1 on smithi078 2024-09-17T09:44:22.818 INFO:teuthology.orchestra.run.smithi078.stdout: 5 199711 10.71 MB/sec execute 317 sec latency 46.906 ms 2024-09-17T09:44:22.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:22 smithi136 ceph-mon[49560]: Upgrade: Updating osd.1 2024-09-17T09:44:22.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:22 smithi136 ceph-mon[49560]: Deploying daemon osd.1 on smithi078 2024-09-17T09:44:23.575 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:23 smithi078 ceph-mon[63787]: pgmap v98: 97 pgs: 97 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.1 MiB/s wr, 401 op/s; 0 B/s, 4 objects/s recovering 2024-09-17T09:44:23.576 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:23 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:23.576 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:23 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:44:23.818 INFO:teuthology.orchestra.run.smithi078.stdout: 5 199994 10.70 MB/sec execute 318 sec latency 52.022 ms 2024-09-17T09:44:23.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:23 smithi136 ceph-mon[49560]: pgmap v98: 97 pgs: 97 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.1 MiB/s wr, 401 op/s; 0 B/s, 4 objects/s recovering 2024-09-17T09:44:23.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:23 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:23.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:23 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:44:24.595 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:24 smithi078 ceph-mon[63787]: pgmap v99: 97 pgs: 97 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.0 MiB/s wr, 611 op/s; 0 B/s, 3 objects/s recovering 2024-09-17T09:44:24.818 INFO:teuthology.orchestra.run.smithi078.stdout: 5 200293 10.69 MB/sec execute 319 sec latency 56.072 ms 2024-09-17T09:44:24.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:24 smithi136 ceph-mon[49560]: pgmap v99: 97 pgs: 97 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.0 MiB/s wr, 611 op/s; 0 B/s, 3 objects/s recovering 2024-09-17T09:44:25.818 INFO:teuthology.orchestra.run.smithi078.stdout: 5 200642 10.66 MB/sec execute 320 sec latency 60.370 ms 2024-09-17T09:44:26.818 INFO:teuthology.orchestra.run.smithi078.stdout: 5 200980 10.67 MB/sec execute 321 sec latency 64.221 ms 2024-09-17T09:44:26.899 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:26 smithi078 systemd[1]: Stopping Ceph osd.1 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:44:27.155 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:27 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1[38202]: 2024-09-17T09:44:27.091+0000 7efec0995700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:44:27.155 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:27 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1[38202]: 2024-09-17T09:44:27.091+0000 7efec0995700 -1 osd.1 93 *** Got signal Terminated *** 2024-09-17T09:44:27.155 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:27 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1[38202]: 2024-09-17T09:44:27.091+0000 7efec0995700 -1 osd.1 93 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:44:27.663 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:27 smithi078 ceph-mon[63787]: pgmap v100: 97 pgs: 97 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.4 MiB/s wr, 545 op/s; 0 B/s, 3 objects/s recovering 2024-09-17T09:44:27.664 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:27 smithi078 ceph-mon[63787]: osd.1 marked itself down and dead 2024-09-17T09:44:27.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:27 smithi136 ceph-mon[49560]: pgmap v100: 97 pgs: 97 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.4 MiB/s wr, 545 op/s; 0 B/s, 3 objects/s recovering 2024-09-17T09:44:27.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:27 smithi136 ceph-mon[49560]: osd.1 marked itself down and dead 2024-09-17T09:44:27.818 INFO:teuthology.orchestra.run.smithi078.stdout: 5 201280 10.66 MB/sec execute 322 sec latency 201.568 ms 2024-09-17T09:44:28.474 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:28 smithi078 podman[70599]: 2024-09-17 09:44:28.096481267 +0000 UTC m=+1.123389174 container died 351873014de4653bbd4d387bff9d7b82a512cc35eca8f3ad8363d0f3fe7a2bca (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, RELEASE=HEAD, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, name=centos-stream, GIT_CLEAN=True, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8) 2024-09-17T09:44:28.725 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:28 smithi078 ceph-mon[63787]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:44:28.725 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:28 smithi078 ceph-mon[63787]: osdmap e94: 8 total, 7 up, 8 in 2024-09-17T09:44:28.725 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:28 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:28.725 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:28 smithi078 ceph-mon[63787]: osdmap e95: 8 total, 7 up, 8 in 2024-09-17T09:44:28.725 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:28 smithi078 podman[70599]: 2024-09-17 09:44:28.721511449 +0000 UTC m=+1.748419355 container cleanup 351873014de4653bbd4d387bff9d7b82a512cc35eca8f3ad8363d0f3fe7a2bca (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, io.buildah.version=1.19.8, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container) 2024-09-17T09:44:28.725 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:28 smithi078 bash[70599]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1 2024-09-17T09:44:28.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:28 smithi136 ceph-mon[49560]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:44:28.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:28 smithi136 ceph-mon[49560]: osdmap e94: 8 total, 7 up, 8 in 2024-09-17T09:44:28.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:28 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:28.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:28 smithi136 ceph-mon[49560]: osdmap e95: 8 total, 7 up, 8 in 2024-09-17T09:44:28.818 INFO:teuthology.orchestra.run.smithi078.stdout: 5 201367 10.63 MB/sec execute 323 sec latency 872.476 ms 2024-09-17T09:44:29.239 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:28 smithi078 podman[70614]: 2024-09-17 09:44:28.981848312 +0000 UTC m=+0.886721171 container remove 351873014de4653bbd4d387bff9d7b82a512cc35eca8f3ad8363d0f3fe7a2bca (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1, release=754, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, vcs-type=git, vendor=Red Hat, Inc., distribution-scope=public, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True) 2024-09-17T09:44:29.503 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:29 smithi078 ceph-mon[63787]: pgmap v102: 97 pgs: 14 stale+active+clean, 83 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.4 MiB/s wr, 545 op/s; 0 B/s, 3 objects/s recovering 2024-09-17T09:44:29.504 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:29 smithi078 podman[70701]: 2024-09-17 09:44:29.35847148 +0000 UTC m=+0.020286619 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:44:29.504 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:29 smithi078 podman[70701]: 2024-09-17 09:44:29.483144401 +0000 UTC m=+0.144959532 container create 398f1fe76a5b75cc01a1be0fbb5dacecf2be643653cabc0db402fe695d50e0e8 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD) 2024-09-17T09:44:29.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:29 smithi136 ceph-mon[49560]: pgmap v102: 97 pgs: 14 stale+active+clean, 83 active+clean; 335 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.4 MiB/s wr, 545 op/s; 0 B/s, 3 objects/s recovering 2024-09-17T09:44:29.818 INFO:teuthology.orchestra.run.smithi078.stdout: 5 201697 10.65 MB/sec execute 324 sec latency 57.593 ms 2024-09-17T09:44:30.016 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:29 smithi078 podman[70701]: 2024-09-17 09:44:29.864017892 +0000 UTC m=+0.525833039 container init 398f1fe76a5b75cc01a1be0fbb5dacecf2be643653cabc0db402fe695d50e0e8 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T09:44:30.017 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:29 smithi078 podman[70701]: 2024-09-17 09:44:29.874471932 +0000 UTC m=+0.536287063 container start 398f1fe76a5b75cc01a1be0fbb5dacecf2be643653cabc0db402fe695d50e0e8 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-17T09:44:30.017 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:29 smithi078 podman[70701]: 2024-09-17 09:44:29.987647411 +0000 UTC m=+0.649462577 container attach 398f1fe76a5b75cc01a1be0fbb5dacecf2be643653cabc0db402fe695d50e0e8 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-17T09:44:30.210 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:30 smithi078 podman[70701]: 2024-09-17 09:44:30.206868952 +0000 UTC m=+0.868684090 container died 398f1fe76a5b75cc01a1be0fbb5dacecf2be643653cabc0db402fe695d50e0e8 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-17T09:44:30.643 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:30 smithi078 ceph-mon[63787]: pgmap v104: 97 pgs: 2 active+undersized, 5 activating+undersized, 30 peering, 2 active+undersized+degraded, 58 active+clean; 336 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 4.7 MiB/s wr, 528 op/s; 36/1545 objects degraded (2.330%) 2024-09-17T09:44:30.643 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:30 smithi078 ceph-mon[63787]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-09-17T09:44:30.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:30 smithi078 ceph-mon[63787]: Health check failed: Degraded data redundancy: 36/1545 objects degraded (2.330%), 2 pgs degraded (PG_DEGRADED) 2024-09-17T09:44:30.818 INFO:teuthology.orchestra.run.smithi078.stdout: 5 202437 10.70 MB/sec execute 325 sec latency 29.880 ms 2024-09-17T09:44:30.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:30 smithi136 ceph-mon[49560]: pgmap v104: 97 pgs: 2 active+undersized, 5 activating+undersized, 30 peering, 2 active+undersized+degraded, 58 active+clean; 336 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 4.7 MiB/s wr, 528 op/s; 36/1545 objects degraded (2.330%) 2024-09-17T09:44:30.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:30 smithi136 ceph-mon[49560]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-09-17T09:44:30.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:30 smithi136 ceph-mon[49560]: Health check failed: Degraded data redundancy: 36/1545 objects degraded (2.330%), 2 pgs degraded (PG_DEGRADED) 2024-09-17T09:44:31.652 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:31 smithi078 podman[70724]: 2024-09-17 09:44:31.466782128 +0000 UTC m=+1.251768297 container remove 398f1fe76a5b75cc01a1be0fbb5dacecf2be643653cabc0db402fe695d50e0e8 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-17T09:44:31.652 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:31 smithi078 systemd[1]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.1.service: Deactivated successfully. 2024-09-17T09:44:31.652 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:31 smithi078 systemd[1]: Stopped Ceph osd.1 for 35c0cd22-74d7-11ef-bceb-c7b262605968. 2024-09-17T09:44:31.652 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:31 smithi078 systemd[1]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.1.service: Consumed 3min 34.842s CPU time. 2024-09-17T09:44:31.818 INFO:teuthology.orchestra.run.smithi078.stdout: 5 203092 10.70 MB/sec execute 326 sec latency 22.003 ms 2024-09-17T09:44:31.902 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:31 smithi078 systemd[1]: Starting Ceph osd.1 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:44:32.404 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:32 smithi078 podman[70824]: 2024-09-17 09:44:32.07538155 +0000 UTC m=+0.032172252 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:44:32.405 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:32 smithi078 podman[70824]: 2024-09-17 09:44:32.250009513 +0000 UTC m=+0.206800202 container create 6face9b8eafc3a83e4512f1fcc59062e8a5a7cd4c6beb2ace0405af5b3404595 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-17T09:44:32.818 INFO:teuthology.orchestra.run.smithi078.stdout: 5 203811 10.72 MB/sec execute 327 sec latency 44.443 ms 2024-09-17T09:44:32.908 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:32 smithi078 podman[70824]: 2024-09-17 09:44:32.701708605 +0000 UTC m=+0.658499300 container init 6face9b8eafc3a83e4512f1fcc59062e8a5a7cd4c6beb2ace0405af5b3404595 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True) 2024-09-17T09:44:32.909 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:32 smithi078 podman[70824]: 2024-09-17 09:44:32.714626667 +0000 UTC m=+0.671417351 container start 6face9b8eafc3a83e4512f1fcc59062e8a5a7cd4c6beb2ace0405af5b3404595 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1-activate, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-17T09:44:32.909 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:32 smithi078 podman[70824]: 2024-09-17 09:44:32.864019732 +0000 UTC m=+0.820810424 container attach 6face9b8eafc3a83e4512f1fcc59062e8a5a7cd4c6beb2ace0405af5b3404595 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T09:44:33.165 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:32 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1-activate[70838]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:44:33.165 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:32 smithi078 bash[70824]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:44:33.165 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:32 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1-activate[70838]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:44:33.165 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:32 smithi078 bash[70824]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:44:33.675 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:33 smithi078 ceph-mon[63787]: pgmap v105: 97 pgs: 2 active+undersized, 5 activating+undersized, 30 peering, 2 active+undersized+degraded, 58 active+clean; 336 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3.1 KiB/s rd, 2.0 MiB/s wr, 244 op/s; 36/1545 objects degraded (2.330%) 2024-09-17T09:44:33.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:33 smithi136 ceph-mon[49560]: pgmap v105: 97 pgs: 2 active+undersized, 5 activating+undersized, 30 peering, 2 active+undersized+degraded, 58 active+clean; 336 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 3.1 KiB/s rd, 2.0 MiB/s wr, 244 op/s; 36/1545 objects degraded (2.330%) 2024-09-17T09:44:33.818 INFO:teuthology.orchestra.run.smithi078.stdout: 5 204264 10.71 MB/sec execute 328 sec latency 38.647 ms 2024-09-17T09:44:34.179 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:34 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1-activate[70838]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-17T09:44:34.179 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:34 smithi078 bash[70824]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-17T09:44:34.179 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:34 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1-activate[70838]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-17T09:44:34.180 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:34 smithi078 bash[70824]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-17T09:44:34.689 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:34 smithi078 ceph-mon[63787]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2024-09-17T09:44:34.690 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:34 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1-activate[70838]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-17T09:44:34.690 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:34 smithi078 bash[70824]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-17T09:44:34.690 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:34 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1-activate[70838]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-17T09:44:34.690 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:34 smithi078 bash[70824]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-17T09:44:34.690 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:34 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1-activate[70838]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-17T09:44:34.690 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:34 smithi078 bash[70824]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-17T09:44:34.690 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:34 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1-activate[70838]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-17T09:44:34.690 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:34 smithi078 bash[70824]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-17T09:44:34.690 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:34 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1-activate[70838]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-17T09:44:34.690 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:34 smithi078 bash[70824]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-17T09:44:34.691 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:34 smithi078 podman[70824]: 2024-09-17 09:44:34.590240578 +0000 UTC m=+2.547031268 container died 6face9b8eafc3a83e4512f1fcc59062e8a5a7cd4c6beb2ace0405af5b3404595 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1-activate, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T09:44:34.818 INFO:teuthology.orchestra.run.smithi078.stdout: 5 204766 10.72 MB/sec execute 329 sec latency 32.636 ms 2024-09-17T09:44:34.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:34 smithi136 ceph-mon[49560]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2024-09-17T09:44:35.732 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:35 smithi078 ceph-mon[63787]: pgmap v106: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.6 KiB/s rd, 7.2 MiB/s wr, 754 op/s; 178/1683 objects degraded (10.576%) 2024-09-17T09:44:35.818 INFO:teuthology.orchestra.run.smithi078.stdout: 5 205232 10.74 MB/sec execute 330 sec latency 33.928 ms 2024-09-17T09:44:35.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:35 smithi136 ceph-mon[49560]: pgmap v106: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.6 KiB/s rd, 7.2 MiB/s wr, 754 op/s; 178/1683 objects degraded (10.576%) 2024-09-17T09:44:35.983 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:35 smithi078 podman[70986]: 2024-09-17 09:44:35.789640476 +0000 UTC m=+1.192353472 container remove 6face9b8eafc3a83e4512f1fcc59062e8a5a7cd4c6beb2ace0405af5b3404595 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1-activate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-17T09:44:36.256 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:36 smithi078 podman[71044]: 2024-09-17 09:44:35.929029458 +0000 UTC m=+0.022060026 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:44:36.257 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:36 smithi078 podman[71044]: 2024-09-17 09:44:36.073127141 +0000 UTC m=+0.166157697 container create eda32560334bf3fa80e0714a87e3774c929c17f1e9f04ab098c87a138d64648e (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-17T09:44:36.511 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:36 smithi078 podman[71044]: 2024-09-17 09:44:36.420025464 +0000 UTC m=+0.513056018 container init eda32560334bf3fa80e0714a87e3774c929c17f1e9f04ab098c87a138d64648e (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-17T09:44:36.512 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:36 smithi078 podman[71044]: 2024-09-17 09:44:36.426687419 +0000 UTC m=+0.519717974 container start eda32560334bf3fa80e0714a87e3774c929c17f1e9f04ab098c87a138d64648e (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-17T09:44:36.512 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:36 smithi078 ceph-osd[71062]: -- 172.21.15.78:0/1343004944 <== mon.0 v2:172.21.15.78:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f4f16121e0 con 0x55f4f07adc00 2024-09-17T09:44:36.773 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:36 smithi078 bash[71044]: eda32560334bf3fa80e0714a87e3774c929c17f1e9f04ab098c87a138d64648e 2024-09-17T09:44:36.774 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:36 smithi078 systemd[1]: Started Ceph osd.1 for 35c0cd22-74d7-11ef-bceb-c7b262605968. 2024-09-17T09:44:36.818 INFO:teuthology.orchestra.run.smithi078.stdout: 5 205599 10.75 MB/sec execute 331 sec latency 49.514 ms 2024-09-17T09:44:37.544 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:37 smithi078 ceph-mon[63787]: pgmap v107: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.5 MiB/s wr, 674 op/s; 178/1683 objects degraded (10.576%) 2024-09-17T09:44:37.544 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:37 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:37.544 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:37 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:37.544 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:37 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:44:37.544 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:37 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 178/1683 objects degraded (10.576%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T09:44:37.818 INFO:teuthology.orchestra.run.smithi078.stdout: 5 205995 10.77 MB/sec execute 332 sec latency 32.998 ms 2024-09-17T09:44:37.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:37 smithi136 ceph-mon[49560]: pgmap v107: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.5 MiB/s wr, 674 op/s; 178/1683 objects degraded (10.576%) 2024-09-17T09:44:37.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:37 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:37.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:37 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:37.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:37 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:44:37.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:37 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 178/1683 objects degraded (10.576%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T09:44:38.588 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:38 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1[71058]: 2024-09-17T09:44:38.571+0000 7fb9ea6c9640 -1 Falling back to public interface 2024-09-17T09:44:38.819 INFO:teuthology.orchestra.run.smithi078.stdout: 5 206228 10.74 MB/sec execute 333 sec latency 58.230 ms 2024-09-17T09:44:38.841 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:38.841 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:44:38.841 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:38 smithi078 ceph-mon[63787]: pgmap v108: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.8 MiB/s wr, 603 op/s; 178/1683 objects degraded (10.576%) 2024-09-17T09:44:38.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:38.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:44:38.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:38 smithi136 ceph-mon[49560]: pgmap v108: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.8 MiB/s wr, 603 op/s; 178/1683 objects degraded (10.576%) 2024-09-17T09:44:39.818 INFO:teuthology.orchestra.run.smithi078.stdout: 5 206465 10.72 MB/sec execute 334 sec latency 51.336 ms 2024-09-17T09:44:40.819 INFO:teuthology.orchestra.run.smithi078.stdout: 5 206728 10.70 MB/sec execute 335 sec latency 88.821 ms 2024-09-17T09:44:41.640 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:41 smithi078 ceph-mon[63787]: pgmap v109: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.8 MiB/s wr, 665 op/s; 194/1716 objects degraded (11.305%) 2024-09-17T09:44:41.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:41 smithi136 ceph-mon[49560]: pgmap v109: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.8 MiB/s wr, 665 op/s; 194/1716 objects degraded (11.305%) 2024-09-17T09:44:41.819 INFO:teuthology.orchestra.run.smithi078.stdout: 5 207023 10.69 MB/sec execute 336 sec latency 54.672 ms 2024-09-17T09:44:42.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:42 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 194/1716 objects degraded (11.305%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T09:44:42.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:42 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 194/1716 objects degraded (11.305%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T09:44:42.819 INFO:teuthology.orchestra.run.smithi078.stdout: 5 207408 10.69 MB/sec execute 337 sec latency 59.976 ms 2024-09-17T09:44:43.656 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:43 smithi078 ceph-mon[63787]: pgmap v110: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.4 MiB/s wr, 577 op/s; 194/1716 objects degraded (11.305%) 2024-09-17T09:44:43.728 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:43 smithi136 ceph-mon[49560]: pgmap v110: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.4 MiB/s wr, 577 op/s; 194/1716 objects degraded (11.305%) 2024-09-17T09:44:43.819 INFO:teuthology.orchestra.run.smithi078.stdout: 5 207812 10.67 MB/sec execute 338 sec latency 53.589 ms 2024-09-17T09:44:44.819 INFO:teuthology.orchestra.run.smithi078.stdout: 5 208187 10.67 MB/sec execute 339 sec latency 48.013 ms 2024-09-17T09:44:45.681 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:45 smithi078 ceph-mon[63787]: pgmap v111: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 336 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 779 op/s; 167/1548 objects degraded (10.788%) 2024-09-17T09:44:45.681 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:45 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:45.682 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:45 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:45.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:45 smithi136 ceph-mon[49560]: pgmap v111: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 336 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 779 op/s; 167/1548 objects degraded (10.788%) 2024-09-17T09:44:45.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:45 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:45.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:45 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:45.819 INFO:teuthology.orchestra.run.smithi078.stdout: 5 208501 10.67 MB/sec execute 340 sec latency 58.996 ms 2024-09-17T09:44:46.202 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:46 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1[71058]: 2024-09-17T09:44:46.156+0000 7fb9ea6c9640 -1 osd.1 0 read_superblock omap replica is missing. 2024-09-17T09:44:46.819 INFO:teuthology.orchestra.run.smithi078.stdout: 5 208810 10.68 MB/sec execute 341 sec latency 54.206 ms 2024-09-17T09:44:47.479 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:47 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1[71058]: 2024-09-17T09:44:47.256+0000 7fb9ea6c9640 -1 osd.1 93 log_to_monitors true 2024-09-17T09:44:47.479 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:44:47 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1[71058]: 2024-09-17T09:44:47.337+0000 7fb9e2473640 -1 osd.1 93 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T09:44:47.480 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:47 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:47.480 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:47 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:47.480 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:47 smithi078 ceph-mon[63787]: pgmap v112: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 336 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.4 MiB/s wr, 439 op/s; 167/1548 objects degraded (10.788%) 2024-09-17T09:44:47.480 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:47 smithi078 ceph-mon[63787]: from='osd.1 [v2:172.21.15.78:6810/3541547585,v1:172.21.15.78:6811/3541547585]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-17T09:44:47.528 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:47 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:47.528 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:47 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:47.528 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:47 smithi136 ceph-mon[49560]: pgmap v112: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 336 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.4 MiB/s wr, 439 op/s; 167/1548 objects degraded (10.788%) 2024-09-17T09:44:47.528 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:47 smithi136 ceph-mon[49560]: from='osd.1 [v2:172.21.15.78:6810/3541547585,v1:172.21.15.78:6811/3541547585]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-17T09:44:47.819 INFO:teuthology.orchestra.run.smithi078.stdout: 5 209116 10.68 MB/sec execute 342 sec latency 233.852 ms 2024-09-17T09:44:48.513 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:48 smithi078 ceph-mon[63787]: from='osd.1 [v2:172.21.15.78:6810/3541547585,v1:172.21.15.78:6811/3541547585]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-17T09:44:48.514 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:48 smithi078 ceph-mon[63787]: osdmap e96: 8 total, 7 up, 8 in 2024-09-17T09:44:48.514 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:48 smithi078 ceph-mon[63787]: from='osd.1 [v2:172.21.15.78:6810/3541547585,v1:172.21.15.78:6811/3541547585]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]} : dispatch 2024-09-17T09:44:48.514 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:48 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 167/1548 objects degraded (10.788%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T09:44:48.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:48 smithi136 ceph-mon[49560]: from='osd.1 [v2:172.21.15.78:6810/3541547585,v1:172.21.15.78:6811/3541547585]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-17T09:44:48.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:48 smithi136 ceph-mon[49560]: osdmap e96: 8 total, 7 up, 8 in 2024-09-17T09:44:48.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:48 smithi136 ceph-mon[49560]: from='osd.1 [v2:172.21.15.78:6810/3541547585,v1:172.21.15.78:6811/3541547585]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]} : dispatch 2024-09-17T09:44:48.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:48 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 167/1548 objects degraded (10.788%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T09:44:48.819 INFO:teuthology.orchestra.run.smithi078.stdout: 5 209263 10.67 MB/sec execute 343 sec latency 671.548 ms 2024-09-17T09:44:49.570 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:49 smithi078 ceph-mon[63787]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:44:49.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:49 smithi078 ceph-mon[63787]: osd.1 [v2:172.21.15.78:6810/3541547585,v1:172.21.15.78:6811/3541547585] boot 2024-09-17T09:44:49.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:49 smithi078 ceph-mon[63787]: osdmap e97: 8 total, 8 up, 8 in 2024-09-17T09:44:49.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:49 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-17T09:44:49.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:49 smithi078 ceph-mon[63787]: pgmap v115: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 2.3 MiB/s wr, 302 op/s; 167/1548 objects degraded (10.788%) 2024-09-17T09:44:49.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:49 smithi078 ceph-mon[63787]: osdmap e98: 8 total, 8 up, 8 in 2024-09-17T09:44:49.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:49 smithi136 ceph-mon[49560]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:44:49.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:49 smithi136 ceph-mon[49560]: osd.1 [v2:172.21.15.78:6810/3541547585,v1:172.21.15.78:6811/3541547585] boot 2024-09-17T09:44:49.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:49 smithi136 ceph-mon[49560]: osdmap e97: 8 total, 8 up, 8 in 2024-09-17T09:44:49.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:49 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-17T09:44:49.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:49 smithi136 ceph-mon[49560]: pgmap v115: 97 pgs: 16 active+undersized, 23 active+undersized+degraded, 58 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 2.3 MiB/s wr, 302 op/s; 167/1548 objects degraded (10.788%) 2024-09-17T09:44:49.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:49 smithi136 ceph-mon[49560]: osdmap e98: 8 total, 8 up, 8 in 2024-09-17T09:44:49.819 INFO:teuthology.orchestra.run.smithi078.stdout: 5 209345 10.64 MB/sec execute 344 sec latency 1671.577 ms 2024-09-17T09:44:50.819 INFO:teuthology.orchestra.run.smithi078.stdout: 5 209345 10.61 MB/sec execute 345 sec latency 2671.606 ms 2024-09-17T09:44:51.565 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:51 smithi078 ceph-mon[63787]: osdmap e99: 8 total, 8 up, 8 in 2024-09-17T09:44:51.565 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:51 smithi078 ceph-mon[63787]: pgmap v118: 97 pgs: 1 remapped+peering, 2 activating, 22 peering, 5 active+undersized, 9 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 6.2 MiB/s wr, 512 op/s; 55/1638 objects degraded (3.358%) 2024-09-17T09:44:51.643 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-17T09:44:51.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:51 smithi136 ceph-mon[49560]: osdmap e99: 8 total, 8 up, 8 in 2024-09-17T09:44:51.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:51 smithi136 ceph-mon[49560]: pgmap v118: 97 pgs: 1 remapped+peering, 2 activating, 22 peering, 5 active+undersized, 9 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 6.2 MiB/s wr, 512 op/s; 55/1638 objects degraded (3.358%) 2024-09-17T09:44:51.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 209443 10.59 MB/sec execute 346 sec latency 3158.654 ms 2024-09-17T09:44:52.581 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:52.581 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:52.581 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:44:52.581 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:44:52.581 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:52.581 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:52.581 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:44:52.581 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:44:52.581 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:44:52.582 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:44:52.582 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-17T09:44:52.582 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:52 smithi078 ceph-mon[63787]: osdmap e100: 8 total, 8 up, 8 in 2024-09-17T09:44:52.582 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:52 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 55/1638 objects degraded (3.358%), 9 pgs degraded (PG_DEGRADED) 2024-09-17T09:44:52.589 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:44:52.589 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (5m) 7s ago 14m 17.6M - 0.27.0 11f11916f8cd 97db2373ce92 2024-09-17T09:44:52.590 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (112s) 7s ago 14m 6899k - 19.3.0-4963-ge646f729 f271290a0950 da528c5ca445 2024-09-17T09:44:52.590 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi136 smithi136 running (106s) 102s ago 13m 6899k - 19.3.0-4963-ge646f729 f271290a0950 909f405348cc 2024-09-17T09:44:52.590 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (5m) 7s ago 14m 52.5M - dad864ee21e9 db74b0f8e0a7 2024-09-17T09:44:52.590 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.jdrsup smithi078 running (8m) 7s ago 8m 39.1M - 17.2.0 e1d6a67b021e 8f834e15ba88 2024-09-17T09:44:52.590 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi136.smslff smithi136 running (8m) 102s ago 8m 15.8M - 17.2.0 e1d6a67b021e 0c20e2b328d3 2024-09-17T09:44:52.590 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rsqofi smithi078 *:8443,9283,8765 running (4m) 7s ago 15m 516M - 19.3.0-4963-ge646f729 f271290a0950 7dda3108c9a0 2024-09-17T09:44:52.591 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi136.hicmlm smithi136 *:8443,9283,8765 running (3m) 102s ago 13m 453M - 19.3.0-4963-ge646f729 f271290a0950 fc518b58e07a 2024-09-17T09:44:52.591 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (3m) 7s ago 15m 50.8M 2048M 19.3.0-4963-ge646f729 f271290a0950 3ead101d3db1 2024-09-17T09:44:52.591 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi136 smithi136 running (2m) 102s ago 13m 36.0M 2048M 19.3.0-4963-ge646f729 f271290a0950 4fce88842b79 2024-09-17T09:44:52.591 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.tcwbvi smithi078 *:2049 running (7m) 7s ago 7m 285M - 4.0 e1d6a67b021e 30c87cb46555 2024-09-17T09:44:52.591 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi136.regjvc smithi136 *:2049 running (7m) 102s ago 7m 87.7M - 4.0 e1d6a67b021e b88e4dc24376 2024-09-17T09:44:52.591 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (5m) 7s ago 14m 10.5M - 1.7.0 72c9c2088986 52dad3adeeaa 2024-09-17T09:44:52.592 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi136 smithi136 *:9100 running (4m) 102s ago 13m 11.9M - 1.7.0 72c9c2088986 f2bc5856e35a 2024-09-17T09:44:52.592 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (65s) 7s ago 12m 246M 1936M 19.3.0-4963-ge646f729 f271290a0950 1ea81cc809b8 2024-09-17T09:44:52.592 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (16s) 7s ago 12m 14.3M 1936M 19.3.0-4963-ge646f729 f271290a0950 eda32560334b 2024-09-17T09:44:52.592 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (11m) 7s ago 11m 480M 1936M 17.2.0 e1d6a67b021e 07eb71e4fa57 2024-09-17T09:44:52.592 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (11m) 7s ago 11m 408M 1936M 17.2.0 e1d6a67b021e 1b3a8be89881 2024-09-17T09:44:52.592 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi136 running (11m) 102s ago 11m 557M 2704M 17.2.0 e1d6a67b021e a0c5b4de2414 2024-09-17T09:44:52.592 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi136 running (10m) 102s ago 10m 383M 2704M 17.2.0 e1d6a67b021e 30a3049955f9 2024-09-17T09:44:52.592 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi136 running (10m) 102s ago 10m 572M 2704M 17.2.0 e1d6a67b021e 1698e1567c33 2024-09-17T09:44:52.592 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi136 running (9m) 102s ago 9m 453M 2704M 17.2.0 e1d6a67b021e 5d3b08cecd78 2024-09-17T09:44:52.593 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (3m) 7s ago 13m 53.7M - 2.51.0 1d3b7f56885b 6e5e8c721eb0 2024-09-17T09:44:52.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:52 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:52.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:52 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:52.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:52 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:44:52.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:52 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:44:52.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:52 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:52.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:52 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:52.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:52 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:44:52.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:52 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:44:52.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:52 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:44:52.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:52 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:44:52.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:52 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-17T09:44:52.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:52 smithi136 ceph-mon[49560]: osdmap e100: 8 total, 8 up, 8 in 2024-09-17T09:44:52.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:52 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 55/1638 objects degraded (3.358%), 9 pgs degraded (PG_DEGRADED) 2024-09-17T09:44:52.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 209670 10.57 MB/sec execute 347 sec latency 432.229 ms 2024-09-17T09:44:53.247 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:44:53.247 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-17T09:44:53.247 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:44:53.247 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:44:53.247 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-17T09:44:53.247 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:44:53.248 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:44:53.248 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-17T09:44:53.248 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-17T09:44:53.248 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:44:53.248 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:44:53.248 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-17T09:44:53.248 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:44:53.248 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:44:53.248 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-17T09:44:53.248 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-09-17T09:44:53.248 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 6 2024-09-17T09:44:53.248 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-17T09:44:53.248 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:44:53.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:53 smithi078 ceph-mon[63787]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:44:53.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:53 smithi078 ceph-mon[63787]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T09:44:53.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:53 smithi078 ceph-mon[63787]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-09-17T09:44:53.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:53 smithi078 ceph-mon[63787]: from='client.34258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:44:53.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:53 smithi078 ceph-mon[63787]: pgmap v120: 97 pgs: 1 remapped+peering, 2 activating, 22 peering, 2 active+undersized+degraded, 70 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 6.1 MiB/s wr, 526 op/s; 25/1641 objects degraded (1.523%); 21 B/s, 59 keys/s, 6 objects/s recovering 2024-09-17T09:44:53.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:53 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:53.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:53 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:44:53.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:53 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/3315798652' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:44:53.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:53 smithi078 ceph-mon[63787]: osdmap e101: 8 total, 8 up, 8 in 2024-09-17T09:44:53.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:53 smithi136 ceph-mon[49560]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:44:53.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:53 smithi136 ceph-mon[49560]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T09:44:53.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:53 smithi136 ceph-mon[49560]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-09-17T09:44:53.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:53 smithi136 ceph-mon[49560]: from='client.34258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:44:53.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:53 smithi136 ceph-mon[49560]: pgmap v120: 97 pgs: 1 remapped+peering, 2 activating, 22 peering, 2 active+undersized+degraded, 70 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 6.1 MiB/s wr, 526 op/s; 25/1641 objects degraded (1.523%); 21 B/s, 59 keys/s, 6 objects/s recovering 2024-09-17T09:44:53.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:53 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:44:53.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:53 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:44:53.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:53 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/3315798652' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:44:53.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:53 smithi136 ceph-mon[49560]: osdmap e101: 8 total, 8 up, 8 in 2024-09-17T09:44:53.800 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:44:53.800 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407", 2024-09-17T09:44:53.800 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-17T09:44:53.800 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T09:44:53.800 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-17T09:44:53.800 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-09-17T09:44:53.800 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-17T09:44:53.800 INFO:teuthology.orchestra.run.smithi078.stdout: "mon" 2024-09-17T09:44:53.800 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-17T09:44:53.800 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "8/23 daemons upgraded", 2024-09-17T09:44:53.801 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T09:44:53.801 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-17T09:44:53.801 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:44:53.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 209903 10.56 MB/sec execute 348 sec latency 955.986 ms 2024-09-17T09:44:54.538 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_WARN Degraded data redundancy: 25/1641 objects degraded (1.523%), 2 pgs degraded 2024-09-17T09:44:54.539 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 25/1641 objects degraded (1.523%), 2 pgs degraded 2024-09-17T09:44:54.539 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.0 is active+undersized+degraded, acting [2,6] 2024-09-17T09:44:54.539 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.4 is active+undersized+degraded, acting [2,5] 2024-09-17T09:44:54.628 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:54 smithi078 ceph-mon[63787]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:44:54.628 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:54 smithi078 ceph-mon[63787]: osdmap e102: 8 total, 8 up, 8 in 2024-09-17T09:44:54.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:54 smithi136 ceph-mon[49560]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:44:54.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:54 smithi136 ceph-mon[49560]: osdmap e102: 8 total, 8 up, 8 in 2024-09-17T09:44:54.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 210142 10.54 MB/sec execute 349 sec latency 491.422 ms 2024-09-17T09:44:55.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:55 smithi078 ceph-mon[63787]: from='client.44203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:44:55.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:55 smithi078 ceph-mon[63787]: pgmap v123: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 peering, 2 active+undersized+degraded, 88 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.9 MiB/s wr, 164 op/s; 122/1623 objects degraded (7.517%); 1002 KiB/s, 59 keys/s, 26 objects/s recovering 2024-09-17T09:44:55.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:55 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/1663717376' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:44:55.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:55 smithi078 ceph-mon[63787]: osdmap e103: 8 total, 8 up, 8 in 2024-09-17T09:44:55.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:55 smithi136 ceph-mon[49560]: from='client.44203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:44:55.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:55 smithi136 ceph-mon[49560]: pgmap v123: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 peering, 2 active+undersized+degraded, 88 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.9 MiB/s wr, 164 op/s; 122/1623 objects degraded (7.517%); 1002 KiB/s, 59 keys/s, 26 objects/s recovering 2024-09-17T09:44:55.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:55 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/1663717376' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:44:55.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:55 smithi136 ceph-mon[49560]: osdmap e103: 8 total, 8 up, 8 in 2024-09-17T09:44:55.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 210381 10.52 MB/sec execute 350 sec latency 1029.128 ms 2024-09-17T09:44:56.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 210754 10.53 MB/sec execute 351 sec latency 57.898 ms 2024-09-17T09:44:57.702 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:57 smithi078 ceph-mon[63787]: pgmap v125: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 peering, 2 active+undersized+degraded, 88 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.9 MiB/s wr, 164 op/s; 122/1623 objects degraded (7.517%); 1005 KiB/s, 59 keys/s, 26 objects/s recovering 2024-09-17T09:44:57.703 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:57 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 122/1623 objects degraded (7.517%), 7 pgs degraded (PG_DEGRADED) 2024-09-17T09:44:57.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:57 smithi136 ceph-mon[49560]: pgmap v125: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 peering, 2 active+undersized+degraded, 88 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.9 MiB/s wr, 164 op/s; 122/1623 objects degraded (7.517%); 1005 KiB/s, 59 keys/s, 26 objects/s recovering 2024-09-17T09:44:57.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:57 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 122/1623 objects degraded (7.517%), 7 pgs degraded (PG_DEGRADED) 2024-09-17T09:44:57.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 211160 10.51 MB/sec execute 352 sec latency 38.692 ms 2024-09-17T09:44:58.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 211568 10.51 MB/sec execute 353 sec latency 36.821 ms 2024-09-17T09:44:59.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:44:59 smithi078 ceph-mon[63787]: pgmap v126: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 peering, 91 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.3 KiB/s rd, 2.8 MiB/s wr, 294 op/s; 97/1602 objects degraded (6.055%); 686 KiB/s, 21 objects/s recovering 2024-09-17T09:44:59.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:44:59 smithi136 ceph-mon[49560]: pgmap v126: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 peering, 91 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.3 KiB/s rd, 2.8 MiB/s wr, 294 op/s; 97/1602 objects degraded (6.055%); 686 KiB/s, 21 objects/s recovering 2024-09-17T09:44:59.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 211908 10.52 MB/sec execute 354 sec latency 34.830 ms 2024-09-17T09:45:00.773 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:00 smithi078 ceph-mon[63787]: pgmap v127: 97 pgs: 97 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.4 MiB/s wr, 474 op/s; 581 KiB/s, 32 objects/s recovering 2024-09-17T09:45:00.773 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:00 smithi078 ceph-mon[63787]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 97/1602 objects degraded (6.055%), 5 pgs degraded) 2024-09-17T09:45:00.774 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:00 smithi078 ceph-mon[63787]: Cluster is now healthy 2024-09-17T09:45:00.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 212301 10.53 MB/sec execute 355 sec latency 42.581 ms 2024-09-17T09:45:00.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:00 smithi136 ceph-mon[49560]: pgmap v127: 97 pgs: 97 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.4 MiB/s wr, 474 op/s; 581 KiB/s, 32 objects/s recovering 2024-09-17T09:45:00.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:00 smithi136 ceph-mon[49560]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 97/1602 objects degraded (6.055%), 5 pgs degraded) 2024-09-17T09:45:00.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:00 smithi136 ceph-mon[49560]: Cluster is now healthy 2024-09-17T09:45:01.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 212717 10.54 MB/sec execute 356 sec latency 43.832 ms 2024-09-17T09:45:02.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 213012 10.52 MB/sec execute 357 sec latency 42.182 ms 2024-09-17T09:45:03.566 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:03 smithi078 ceph-mon[63787]: pgmap v128: 97 pgs: 97 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.4 MiB/s wr, 365 op/s; 2.8 KiB/s, 20 objects/s recovering 2024-09-17T09:45:03.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:03 smithi136 ceph-mon[49560]: pgmap v128: 97 pgs: 97 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.4 MiB/s wr, 365 op/s; 2.8 KiB/s, 20 objects/s recovering 2024-09-17T09:45:03.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 213394 10.52 MB/sec execute 358 sec latency 46.562 ms 2024-09-17T09:45:04.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 213760 10.52 MB/sec execute 359 sec latency 46.868 ms 2024-09-17T09:45:05.659 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:05 smithi078 ceph-mon[63787]: pgmap v129: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.5 MiB/s wr, 543 op/s; 0 B/s, 14 objects/s recovering 2024-09-17T09:45:05.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:05 smithi136 ceph-mon[49560]: pgmap v129: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.5 MiB/s wr, 543 op/s; 0 B/s, 14 objects/s recovering 2024-09-17T09:45:05.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 214169 10.52 MB/sec execute 360 sec latency 63.288 ms 2024-09-17T09:45:06.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 214548 10.50 MB/sec execute 361 sec latency 52.917 ms 2024-09-17T09:45:07.694 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:07 smithi078 ceph-mon[63787]: pgmap v130: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.9 MiB/s wr, 490 op/s; 0 B/s, 13 objects/s recovering 2024-09-17T09:45:07.694 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:07 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-17T09:45:07.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 214968 10.50 MB/sec execute 362 sec latency 51.644 ms 2024-09-17T09:45:07.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:07 smithi136 ceph-mon[49560]: pgmap v130: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.9 MiB/s wr, 490 op/s; 0 B/s, 13 objects/s recovering 2024-09-17T09:45:07.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:07 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-17T09:45:08.725 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:08 smithi078 ceph-mon[63787]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T09:45:08.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:08 smithi078 ceph-mon[63787]: Upgrade: osd.2 is safe to restart 2024-09-17T09:45:08.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:08 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:08.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:08 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:45:08.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:08 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:08.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 215324 10.50 MB/sec execute 363 sec latency 44.598 ms 2024-09-17T09:45:08.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:08 smithi136 ceph-mon[49560]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T09:45:08.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:08 smithi136 ceph-mon[49560]: Upgrade: osd.2 is safe to restart 2024-09-17T09:45:08.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:08 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:08.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:08 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:45:08.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:08 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:09.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:09 smithi078 ceph-mon[63787]: pgmap v131: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.9 MiB/s wr, 473 op/s; 0 B/s, 12 objects/s recovering 2024-09-17T09:45:09.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 215807 10.53 MB/sec execute 364 sec latency 39.131 ms 2024-09-17T09:45:09.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:09 smithi136 ceph-mon[49560]: pgmap v131: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.9 MiB/s wr, 473 op/s; 0 B/s, 12 objects/s recovering 2024-09-17T09:45:10.745 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:10 smithi078 ceph-mon[63787]: Upgrade: Updating osd.2 2024-09-17T09:45:10.746 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:10 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:10.746 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:10 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-17T09:45:10.746 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:10 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:45:10.746 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:10 smithi078 ceph-mon[63787]: Deploying daemon osd.2 on smithi078 2024-09-17T09:45:10.746 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:10 smithi078 ceph-mon[63787]: pgmap v132: 97 pgs: 97 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.8 MiB/s wr, 584 op/s; 0 B/s, 8 objects/s recovering 2024-09-17T09:45:10.825 INFO:teuthology.orchestra.run.smithi078.stdout: 5 216228 10.53 MB/sec execute 365 sec latency 40.439 ms 2024-09-17T09:45:10.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:10 smithi136 ceph-mon[49560]: Upgrade: Updating osd.2 2024-09-17T09:45:10.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:10 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:10.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:10 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-17T09:45:10.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:10 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:45:10.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:10 smithi136 ceph-mon[49560]: Deploying daemon osd.2 on smithi078 2024-09-17T09:45:10.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:10 smithi136 ceph-mon[49560]: pgmap v132: 97 pgs: 97 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.8 MiB/s wr, 584 op/s; 0 B/s, 8 objects/s recovering 2024-09-17T09:45:11.825 INFO:teuthology.orchestra.run.smithi078.stdout: 5 216552 10.52 MB/sec execute 366 sec latency 53.206 ms 2024-09-17T09:45:12.825 INFO:teuthology.orchestra.run.smithi078.stdout: 5 216870 10.52 MB/sec execute 367 sec latency 35.504 ms 2024-09-17T09:45:13.533 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:13 smithi078 ceph-mon[63787]: pgmap v133: 97 pgs: 97 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.2 MiB/s wr, 549 op/s 2024-09-17T09:45:13.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:13 smithi136 ceph-mon[49560]: pgmap v133: 97 pgs: 97 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.2 MiB/s wr, 549 op/s 2024-09-17T09:45:13.825 INFO:teuthology.orchestra.run.smithi078.stdout: 5 217180 10.51 MB/sec execute 368 sec latency 54.264 ms 2024-09-17T09:45:14.825 INFO:teuthology.orchestra.run.smithi078.stdout: 5 217552 10.51 MB/sec execute 369 sec latency 44.092 ms 2024-09-17T09:45:15.329 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:15 smithi078 systemd[1]: Stopping Ceph osd.2 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:45:15.580 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:15 smithi078 ceph-mon[63787]: pgmap v134: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 7.0 MiB/s wr, 673 op/s 2024-09-17T09:45:15.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:15 smithi136 ceph-mon[49560]: pgmap v134: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 7.0 MiB/s wr, 673 op/s 2024-09-17T09:45:15.825 INFO:teuthology.orchestra.run.smithi078.stdout: 5 217873 10.50 MB/sec execute 370 sec latency 49.559 ms 2024-09-17T09:45:15.836 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:15 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2[41869]: 2024-09-17T09:45:15.644+0000 7feda461b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:45:15.836 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:15 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2[41869]: 2024-09-17T09:45:15.644+0000 7feda461b700 -1 osd.2 103 *** Got signal Terminated *** 2024-09-17T09:45:15.837 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:15 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2[41869]: 2024-09-17T09:45:15.644+0000 7feda461b700 -1 osd.2 103 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:45:16.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:16 smithi078 ceph-mon[63787]: osd.2 marked itself down and dead 2024-09-17T09:45:16.825 INFO:teuthology.orchestra.run.smithi078.stdout: 5 218131 10.48 MB/sec execute 371 sec latency 353.468 ms 2024-09-17T09:45:16.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:16 smithi136 ceph-mon[49560]: osd.2 marked itself down and dead 2024-09-17T09:45:17.477 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:17 smithi078 podman[72950]: 2024-09-17 09:45:17.14165165 +0000 UTC m=+1.662571391 container died 07eb71e4fa579f4d2dc5e8cc69ad1ea5b600a180e2901a7d814e391629300eae (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , distribution-scope=public, io.openshift.expose-services=, GIT_CLEAN=True, architecture=x86_64, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream) 2024-09-17T09:45:17.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:17 smithi078 ceph-mon[63787]: pgmap v135: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.6 MiB/s wr, 462 op/s 2024-09-17T09:45:17.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:17 smithi078 ceph-mon[63787]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:45:17.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:17 smithi078 ceph-mon[63787]: osdmap e104: 8 total, 7 up, 8 in 2024-09-17T09:45:17.825 INFO:teuthology.orchestra.run.smithi078.stdout: 5 218255 10.46 MB/sec execute 372 sec latency 1047.164 ms 2024-09-17T09:45:17.876 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:17 smithi136 ceph-mon[49560]: pgmap v135: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.6 MiB/s wr, 462 op/s 2024-09-17T09:45:17.877 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:17 smithi136 ceph-mon[49560]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:45:17.877 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:17 smithi136 ceph-mon[49560]: osdmap e104: 8 total, 7 up, 8 in 2024-09-17T09:45:18.229 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:18 smithi078 podman[72950]: 2024-09-17 09:45:18.203820815 +0000 UTC m=+2.724740545 container cleanup 07eb71e4fa579f4d2dc5e8cc69ad1ea5b600a180e2901a7d814e391629300eae (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2, io.openshift.tags=base centos centos-stream, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, io.openshift.expose-services=, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, release=754, RELEASE=HEAD, vendor=Red Hat, Inc., distribution-scope=public) 2024-09-17T09:45:18.229 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:18 smithi078 bash[72950]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2 2024-09-17T09:45:18.738 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:18 smithi078 ceph-mon[63787]: osdmap e105: 8 total, 7 up, 8 in 2024-09-17T09:45:18.739 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:18 smithi078 podman[72966]: 2024-09-17 09:45:18.718975413 +0000 UTC m=+1.579719142 container remove 07eb71e4fa579f4d2dc5e8cc69ad1ea5b600a180e2901a7d814e391629300eae (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, version=8, io.k8s.display-name=CentOS Stream 8, vcs-type=git, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, release=754, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , ceph=True, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8) 2024-09-17T09:45:18.825 INFO:teuthology.orchestra.run.smithi078.stdout: 5 218829 10.47 MB/sec execute 373 sec latency 34.330 ms 2024-09-17T09:45:18.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:18 smithi136 ceph-mon[49560]: osdmap e105: 8 total, 7 up, 8 in 2024-09-17T09:45:19.244 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:19 smithi078 podman[73051]: 2024-09-17 09:45:19.125133205 +0000 UTC m=+0.024028458 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:45:19.495 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:19 smithi078 ceph-mon[63787]: pgmap v138: 97 pgs: 1 active+undersized, 6 peering, 3 stale+active+clean, 2 active+undersized+degraded, 85 active+clean; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.1 MiB/s wr, 367 op/s; 25/1605 objects degraded (1.558%) 2024-09-17T09:45:19.495 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:19 smithi078 ceph-mon[63787]: Health check failed: Degraded data redundancy: 25/1605 objects degraded (1.558%), 2 pgs degraded (PG_DEGRADED) 2024-09-17T09:45:19.496 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:19 smithi078 podman[73051]: 2024-09-17 09:45:19.26968932 +0000 UTC m=+0.168584563 container create b99bd0d796b0544511cc2e87ad36ede26e602d1c22e765558fd2e7701aece305 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2-deactivate, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-17T09:45:19.747 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:19 smithi078 podman[73051]: 2024-09-17 09:45:19.68074246 +0000 UTC m=+0.579637732 container init b99bd0d796b0544511cc2e87ad36ede26e602d1c22e765558fd2e7701aece305 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-17T09:45:19.747 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:19 smithi078 podman[73051]: 2024-09-17 09:45:19.693018546 +0000 UTC m=+0.591913784 container start b99bd0d796b0544511cc2e87ad36ede26e602d1c22e765558fd2e7701aece305 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72) 2024-09-17T09:45:19.828 INFO:teuthology.orchestra.run.smithi078.stdout: 5 219445 10.51 MB/sec execute 374 sec latency 32.781 ms 2024-09-17T09:45:19.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:19 smithi136 ceph-mon[49560]: pgmap v138: 97 pgs: 1 active+undersized, 6 peering, 3 stale+active+clean, 2 active+undersized+degraded, 85 active+clean; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.1 MiB/s wr, 367 op/s; 25/1605 objects degraded (1.558%) 2024-09-17T09:45:19.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:19 smithi136 ceph-mon[49560]: Health check failed: Degraded data redundancy: 25/1605 objects degraded (1.558%), 2 pgs degraded (PG_DEGRADED) 2024-09-17T09:45:20.001 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:19 smithi078 podman[73051]: 2024-09-17 09:45:19.849477068 +0000 UTC m=+0.748372331 container attach b99bd0d796b0544511cc2e87ad36ede26e602d1c22e765558fd2e7701aece305 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-17T09:45:20.250 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:20 smithi078 podman[73051]: 2024-09-17 09:45:20.065746927 +0000 UTC m=+0.964642165 container died b99bd0d796b0544511cc2e87ad36ede26e602d1c22e765558fd2e7701aece305 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-17T09:45:20.838 INFO:teuthology.orchestra.run.smithi078.stdout: 5 219966 10.52 MB/sec execute 375 sec latency 32.763 ms 2024-09-17T09:45:21.515 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:21 smithi078 podman[73072]: 2024-09-17 09:45:21.349353193 +0000 UTC m=+1.271348574 container remove b99bd0d796b0544511cc2e87ad36ede26e602d1c22e765558fd2e7701aece305 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.build-date=20240731) 2024-09-17T09:45:21.515 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:21 smithi078 systemd[1]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.2.service: Deactivated successfully. 2024-09-17T09:45:21.515 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:21 smithi078 systemd[1]: Stopped Ceph osd.2 for 35c0cd22-74d7-11ef-bceb-c7b262605968. 2024-09-17T09:45:21.515 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:21 smithi078 systemd[1]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.2.service: Consumed 2min 43.696s CPU time. 2024-09-17T09:45:21.516 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:21 smithi078 ceph-mon[63787]: pgmap v139: 97 pgs: 3 active+undersized, 6 peering, 16 active+undersized+degraded, 72 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.5 MiB/s wr, 491 op/s; 118/1581 objects degraded (7.464%) 2024-09-17T09:45:21.767 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:21 smithi078 systemd[1]: Starting Ceph osd.2 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:45:21.838 INFO:teuthology.orchestra.run.smithi078.stdout: 5 220241 10.50 MB/sec execute 376 sec latency 36.419 ms 2024-09-17T09:45:21.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:21 smithi136 ceph-mon[49560]: pgmap v139: 97 pgs: 3 active+undersized, 6 peering, 16 active+undersized+degraded, 72 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.5 MiB/s wr, 491 op/s; 118/1581 objects degraded (7.464%) 2024-09-17T09:45:22.022 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:21 smithi078 podman[73166]: 2024-09-17 09:45:21.898918184 +0000 UTC m=+0.025678542 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:45:22.276 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:22 smithi078 podman[73166]: 2024-09-17 09:45:22.023487246 +0000 UTC m=+0.150247577 container create 057a0312203f47bd3c55c4ad4d22ee39ee751acddf1030c37809c05bfc0a4d15 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.vendor=CentOS) 2024-09-17T09:45:22.838 INFO:teuthology.orchestra.run.smithi078.stdout: 5 220618 10.50 MB/sec execute 377 sec latency 50.287 ms 2024-09-17T09:45:23.036 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:22 smithi078 podman[73166]: 2024-09-17 09:45:22.786515008 +0000 UTC m=+0.913275364 container init 057a0312203f47bd3c55c4ad4d22ee39ee751acddf1030c37809c05bfc0a4d15 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72) 2024-09-17T09:45:23.036 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:22 smithi078 podman[73166]: 2024-09-17 09:45:22.792025451 +0000 UTC m=+0.918785787 container start 057a0312203f47bd3c55c4ad4d22ee39ee751acddf1030c37809c05bfc0a4d15 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72) 2024-09-17T09:45:23.036 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:22 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2-activate[73180]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:45:23.036 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:22 smithi078 bash[73166]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:45:23.037 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:22 smithi078 podman[73166]: 2024-09-17 09:45:22.98340813 +0000 UTC m=+1.110168468 container attach 057a0312203f47bd3c55c4ad4d22ee39ee751acddf1030c37809c05bfc0a4d15 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-17T09:45:23.037 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:22 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2-activate[73180]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:45:23.037 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:22 smithi078 bash[73166]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:45:23.538 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:23 smithi078 ceph-mon[63787]: pgmap v140: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 2.8 MiB/s wr, 305 op/s; 154/1593 objects degraded (9.667%) 2024-09-17T09:45:23.539 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:23 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:23.539 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:23 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:45:23.838 INFO:teuthology.orchestra.run.smithi078.stdout: 5 220919 10.50 MB/sec execute 378 sec latency 53.463 ms 2024-09-17T09:45:23.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:23 smithi136 ceph-mon[49560]: pgmap v140: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 2.8 MiB/s wr, 305 op/s; 154/1593 objects degraded (9.667%) 2024-09-17T09:45:23.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:23 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:23.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:23 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:45:24.075 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:23 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2-activate[73180]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-17T09:45:24.075 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:23 smithi078 bash[73166]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-17T09:45:24.075 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:23 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2-activate[73180]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-17T09:45:24.075 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:23 smithi078 bash[73166]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-17T09:45:24.332 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:24 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2-activate[73180]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-17T09:45:24.332 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:24 smithi078 bash[73166]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-17T09:45:24.332 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:24 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2-activate[73180]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-17T09:45:24.332 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:24 smithi078 bash[73166]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-17T09:45:24.332 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:24 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2-activate[73180]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-17T09:45:24.332 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:24 smithi078 bash[73166]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-17T09:45:24.332 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:24 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2-activate[73180]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-17T09:45:24.332 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:24 smithi078 bash[73166]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-17T09:45:24.332 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:24 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2-activate[73180]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-17T09:45:24.332 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:24 smithi078 bash[73166]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-17T09:45:24.592 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:24 smithi078 podman[73330]: 2024-09-17 09:45:24.347809959 +0000 UTC m=+0.025883220 container died 057a0312203f47bd3c55c4ad4d22ee39ee751acddf1030c37809c05bfc0a4d15 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2-activate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T09:45:24.593 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:24 smithi078 ceph-mon[63787]: pgmap v141: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.8 MiB/s wr, 683 op/s; 158/1677 objects degraded (9.422%) 2024-09-17T09:45:24.593 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:24 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 158/1677 objects degraded (9.422%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T09:45:24.838 INFO:teuthology.orchestra.run.smithi078.stdout: 5 221288 10.49 MB/sec execute 379 sec latency 49.598 ms 2024-09-17T09:45:24.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:24 smithi136 ceph-mon[49560]: pgmap v141: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.8 MiB/s wr, 683 op/s; 158/1677 objects degraded (9.422%) 2024-09-17T09:45:24.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:24 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 158/1677 objects degraded (9.422%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T09:45:25.130 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-17T09:45:25.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:25 smithi078 ceph-mon[63787]: from='client.34278 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:45:25.627 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:25 smithi078 podman[73330]: 2024-09-17 09:45:25.565870702 +0000 UTC m=+1.243943942 container remove 057a0312203f47bd3c55c4ad4d22ee39ee751acddf1030c37809c05bfc0a4d15 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T09:45:25.838 INFO:teuthology.orchestra.run.smithi078.stdout: 5 221622 10.48 MB/sec execute 380 sec latency 37.357 ms 2024-09-17T09:45:25.882 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:25 smithi078 podman[73437]: 2024-09-17 09:45:25.745960958 +0000 UTC m=+0.025595785 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:45:25.883 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:25 smithi078 podman[73437]: 2024-09-17 09:45:25.866107872 +0000 UTC m=+0.145742688 container create 83a32ad7c358ab77288d7ebbb4c5b7abc55749669b2d48e6265ff6721c955cd1 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.license=GPLv2) 2024-09-17T09:45:25.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:25 smithi136 ceph-mon[49560]: from='client.34278 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:45:26.218 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:45:26.218 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (5m) 40s ago 14m 17.6M - 0.27.0 11f11916f8cd 97db2373ce92 2024-09-17T09:45:26.219 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (2m) 40s ago 14m 6899k - 19.3.0-4963-ge646f729 f271290a0950 da528c5ca445 2024-09-17T09:45:26.219 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi136 smithi136 running (2m) 2m ago 14m 6899k - 19.3.0-4963-ge646f729 f271290a0950 909f405348cc 2024-09-17T09:45:26.219 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (5m) 40s ago 14m 52.5M - dad864ee21e9 db74b0f8e0a7 2024-09-17T09:45:26.219 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.jdrsup smithi078 running (8m) 40s ago 8m 39.1M - 17.2.0 e1d6a67b021e 8f834e15ba88 2024-09-17T09:45:26.219 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi136.smslff smithi136 running (8m) 2m ago 8m 15.8M - 17.2.0 e1d6a67b021e 0c20e2b328d3 2024-09-17T09:45:26.219 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rsqofi smithi078 *:8443,9283,8765 running (4m) 40s ago 16m 516M - 19.3.0-4963-ge646f729 f271290a0950 7dda3108c9a0 2024-09-17T09:45:26.219 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi136.hicmlm smithi136 *:8443,9283,8765 running (4m) 2m ago 14m 453M - 19.3.0-4963-ge646f729 f271290a0950 fc518b58e07a 2024-09-17T09:45:26.219 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (3m) 40s ago 16m 50.8M 2048M 19.3.0-4963-ge646f729 f271290a0950 3ead101d3db1 2024-09-17T09:45:26.219 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi136 smithi136 running (3m) 2m ago 14m 36.0M 2048M 19.3.0-4963-ge646f729 f271290a0950 4fce88842b79 2024-09-17T09:45:26.219 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.tcwbvi smithi078 *:2049 running (8m) 40s ago 8m 285M - 4.0 e1d6a67b021e 30c87cb46555 2024-09-17T09:45:26.219 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi136.regjvc smithi136 *:2049 running (8m) 2m ago 8m 87.7M - 4.0 e1d6a67b021e b88e4dc24376 2024-09-17T09:45:26.219 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (5m) 40s ago 14m 10.5M - 1.7.0 72c9c2088986 52dad3adeeaa 2024-09-17T09:45:26.219 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi136 smithi136 *:9100 running (5m) 2m ago 14m 11.9M - 1.7.0 72c9c2088986 f2bc5856e35a 2024-09-17T09:45:26.220 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (99s) 40s ago 13m 246M 1936M 19.3.0-4963-ge646f729 f271290a0950 1ea81cc809b8 2024-09-17T09:45:26.220 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (50s) 40s ago 12m 14.3M 1936M 19.3.0-4963-ge646f729 f271290a0950 eda32560334b 2024-09-17T09:45:26.220 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (12m) 40s ago 12m 480M 1936M 17.2.0 e1d6a67b021e 07eb71e4fa57 2024-09-17T09:45:26.220 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (11m) 40s ago 11m 408M 1936M 17.2.0 e1d6a67b021e 1b3a8be89881 2024-09-17T09:45:26.220 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi136 running (11m) 2m ago 11m 557M 2704M 17.2.0 e1d6a67b021e a0c5b4de2414 2024-09-17T09:45:26.220 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi136 running (11m) 2m ago 11m 383M 2704M 17.2.0 e1d6a67b021e 30a3049955f9 2024-09-17T09:45:26.220 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi136 running (10m) 2m ago 10m 572M 2704M 17.2.0 e1d6a67b021e 1698e1567c33 2024-09-17T09:45:26.220 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi136 running (10m) 2m ago 10m 453M 2704M 17.2.0 e1d6a67b021e 5d3b08cecd78 2024-09-17T09:45:26.220 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (4m) 40s ago 13m 53.7M - 2.51.0 1d3b7f56885b 6e5e8c721eb0 2024-09-17T09:45:26.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:26 smithi078 ceph-mon[63787]: from='client.44215 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:45:26.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:26 smithi078 ceph-mon[63787]: from='client.34286 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:45:26.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:26 smithi078 ceph-mon[63787]: pgmap v142: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.5 MiB/s wr, 550 op/s; 158/1677 objects degraded (9.422%) 2024-09-17T09:45:26.838 INFO:teuthology.orchestra.run.smithi078.stdout: 5 222047 10.48 MB/sec execute 381 sec latency 43.777 ms 2024-09-17T09:45:26.852 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:45:26.853 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-17T09:45:26.853 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:45:26.853 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:45:26.853 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-17T09:45:26.853 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:45:26.853 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:45:26.853 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-17T09:45:26.853 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-17T09:45:26.853 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:45:26.853 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:45:26.853 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-17T09:45:26.853 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:45:26.854 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:45:26.854 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-17T09:45:26.854 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-17T09:45:26.854 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 6 2024-09-17T09:45:26.854 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-17T09:45:26.854 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:45:26.904 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:26 smithi078 podman[73437]: 2024-09-17 09:45:26.721664062 +0000 UTC m=+1.001298878 container init 83a32ad7c358ab77288d7ebbb4c5b7abc55749669b2d48e6265ff6721c955cd1 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:45:26.904 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:26 smithi078 podman[73437]: 2024-09-17 09:45:26.732999674 +0000 UTC m=+1.012634495 container start 83a32ad7c358ab77288d7ebbb4c5b7abc55749669b2d48e6265ff6721c955cd1 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-17T09:45:26.904 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:26 smithi078 ceph-osd[73496]: -- 172.21.15.78:0/2250892075 <== mon.1 v2:172.21.15.136:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x562c3807e1e0 con 0x562c3807a000 2024-09-17T09:45:26.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:26 smithi136 ceph-mon[49560]: from='client.44215 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:45:26.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:26 smithi136 ceph-mon[49560]: from='client.34286 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:45:26.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:26 smithi136 ceph-mon[49560]: pgmap v142: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.5 MiB/s wr, 550 op/s; 158/1677 objects degraded (9.422%) 2024-09-17T09:45:27.154 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:26 smithi078 bash[73437]: 83a32ad7c358ab77288d7ebbb4c5b7abc55749669b2d48e6265ff6721c955cd1 2024-09-17T09:45:27.154 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:26 smithi078 systemd[1]: Started Ceph osd.2 for 35c0cd22-74d7-11ef-bceb-c7b262605968. 2024-09-17T09:45:27.403 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:45:27.403 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407", 2024-09-17T09:45:27.403 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-17T09:45:27.403 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T09:45:27.403 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-17T09:45:27.403 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-09-17T09:45:27.403 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-17T09:45:27.403 INFO:teuthology.orchestra.run.smithi078.stdout: "mon" 2024-09-17T09:45:27.403 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-17T09:45:27.403 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "8/23 daemons upgraded", 2024-09-17T09:45:27.404 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T09:45:27.404 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-17T09:45:27.404 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:45:27.657 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:27 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/3633931849' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:45:27.657 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:27 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:27.657 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:27 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:27.657 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:27 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:45:27.657 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:27 smithi078 ceph-mon[63787]: from='client.44227 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:45:27.838 INFO:teuthology.orchestra.run.smithi078.stdout: 5 222372 10.47 MB/sec execute 382 sec latency 33.217 ms 2024-09-17T09:45:27.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:27 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/3633931849' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:45:27.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:27 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:27.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:27 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:27.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:27 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:45:27.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:27 smithi136 ceph-mon[49560]: from='client.44227 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:45:28.068 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 158/1677 objects degraded (9.422%), 19 pgs degraded 2024-09-17T09:45:28.069 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-17T09:45:28.069 INFO:teuthology.orchestra.run.smithi078.stdout: osd.2 (root=default,host=smithi078) is down 2024-09-17T09:45:28.069 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 158/1677 objects degraded (9.422%), 19 pgs degraded 2024-09-17T09:45:28.069 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1 is active+undersized+degraded, acting [3,0] 2024-09-17T09:45:28.069 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.3 is active+undersized+degraded, acting [5,4] 2024-09-17T09:45:28.069 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.7 is active+undersized+degraded, acting [6,7] 2024-09-17T09:45:28.069 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.c is active+undersized+degraded, acting [5,0] 2024-09-17T09:45:28.069 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.e is active+undersized+degraded, acting [3,7] 2024-09-17T09:45:28.069 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.10 is active+undersized+degraded, acting [1,0] 2024-09-17T09:45:28.069 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.16 is active+undersized+degraded, acting [5,6] 2024-09-17T09:45:28.069 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.17 is active+undersized+degraded, acting [6,5] 2024-09-17T09:45:28.069 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1c is active+undersized+degraded, acting [7,5] 2024-09-17T09:45:28.070 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.0 is active+undersized+degraded, acting [1,6] 2024-09-17T09:45:28.070 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.4 is active+undersized+degraded, acting [1,5] 2024-09-17T09:45:28.070 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.5 is active+undersized+degraded, acting [5,3] 2024-09-17T09:45:28.070 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.9 is active+undersized+degraded, acting [4,7] 2024-09-17T09:45:28.070 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.13 is active+undersized+degraded, acting [7,4] 2024-09-17T09:45:28.070 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1a is active+undersized+degraded, acting [4,1] 2024-09-17T09:45:28.070 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1e is active+undersized+degraded, acting [3,6] 2024-09-17T09:45:28.070 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1f is active+undersized+degraded, acting [0,5] 2024-09-17T09:45:28.070 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.14 is active+undersized+degraded, acting [3,7] 2024-09-17T09:45:28.070 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.19 is active+undersized+degraded, acting [3,0] 2024-09-17T09:45:28.667 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:28 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2[73492]: 2024-09-17T09:45:28.417+0000 7f1252a47640 -1 Falling back to public interface 2024-09-17T09:45:28.838 INFO:teuthology.orchestra.run.smithi078.stdout: 5 222801 10.49 MB/sec execute 383 sec latency 44.710 ms 2024-09-17T09:45:28.918 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:28 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:28.919 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:28 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/3055938619' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:45:28.919 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:28 smithi078 ceph-mon[63787]: pgmap v143: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 344 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.4 MiB/s wr, 518 op/s; 151/1647 objects degraded (9.168%) 2024-09-17T09:45:29.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:28 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:29.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:28 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/3055938619' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:45:29.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:28 smithi136 ceph-mon[49560]: pgmap v143: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 344 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.4 MiB/s wr, 518 op/s; 151/1647 objects degraded (9.168%) 2024-09-17T09:45:29.839 INFO:teuthology.orchestra.run.smithi078.stdout: 5 223153 10.50 MB/sec execute 384 sec latency 38.995 ms 2024-09-17T09:45:30.760 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:30 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 146/1590 objects degraded (9.182%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T09:45:30.838 INFO:teuthology.orchestra.run.smithi078.stdout: 5 223441 10.49 MB/sec execute 385 sec latency 78.887 ms 2024-09-17T09:45:30.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:30 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 146/1590 objects degraded (9.182%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T09:45:31.778 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:31 smithi078 ceph-mon[63787]: pgmap v144: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.5 MiB/s wr, 667 op/s; 146/1590 objects degraded (9.182%) 2024-09-17T09:45:31.839 INFO:teuthology.orchestra.run.smithi078.stdout: 5 223630 10.47 MB/sec execute 386 sec latency 54.446 ms 2024-09-17T09:45:31.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:31 smithi136 ceph-mon[49560]: pgmap v144: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 340 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.5 MiB/s wr, 667 op/s; 146/1590 objects degraded (9.182%) 2024-09-17T09:45:32.839 INFO:teuthology.orchestra.run.smithi078.stdout: 5 223817 10.45 MB/sec execute 387 sec latency 61.312 ms 2024-09-17T09:45:33.560 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:33 smithi078 ceph-mon[63787]: pgmap v145: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.3 MiB/s wr, 493 op/s; 151/1605 objects degraded (9.408%) 2024-09-17T09:45:33.839 INFO:teuthology.orchestra.run.smithi078.stdout: 5 224152 10.44 MB/sec execute 388 sec latency 68.956 ms 2024-09-17T09:45:33.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:33 smithi136 ceph-mon[49560]: pgmap v145: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 339 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.3 MiB/s wr, 493 op/s; 151/1605 objects degraded (9.408%) 2024-09-17T09:45:34.833 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:34 smithi078 ceph-mon[63787]: pgmap v146: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.7 MiB/s wr, 642 op/s; 154/1614 objects degraded (9.542%) 2024-09-17T09:45:34.838 INFO:teuthology.orchestra.run.smithi078.stdout: 5 224618 10.46 MB/sec execute 389 sec latency 42.171 ms 2024-09-17T09:45:34.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:34 smithi136 ceph-mon[49560]: pgmap v146: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.7 MiB/s wr, 642 op/s; 154/1614 objects degraded (9.542%) 2024-09-17T09:45:35.335 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:35 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2[73492]: 2024-09-17T09:45:35.262+0000 7f1252a47640 -1 osd.2 0 read_superblock omap replica is missing. 2024-09-17T09:45:35.839 INFO:teuthology.orchestra.run.smithi078.stdout: 5 224974 10.45 MB/sec execute 390 sec latency 49.516 ms 2024-09-17T09:45:35.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:35 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:35.853 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:35 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:35.853 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:35 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:35.853 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:35 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:36.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:35 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:36.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:35 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:36.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:35 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:36.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:35 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:36.648 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:36 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2[73492]: 2024-09-17T09:45:36.419+0000 7f1252a47640 -1 osd.2 103 log_to_monitors true 2024-09-17T09:45:36.839 INFO:teuthology.orchestra.run.smithi078.stdout: 5 225466 10.45 MB/sec execute 391 sec latency 42.130 ms 2024-09-17T09:45:36.980 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:36 smithi078 ceph-mon[63787]: from='osd.2 [v2:172.21.15.78:6818/2245003569,v1:172.21.15.78:6819/2245003569]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-17T09:45:36.980 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:36 smithi078 ceph-mon[63787]: pgmap v147: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.1 MiB/s wr, 389 op/s; 154/1614 objects degraded (9.542%) 2024-09-17T09:45:37.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:36 smithi136 ceph-mon[49560]: from='osd.2 [v2:172.21.15.78:6818/2245003569,v1:172.21.15.78:6819/2245003569]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-17T09:45:37.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:36 smithi136 ceph-mon[49560]: pgmap v147: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.1 MiB/s wr, 389 op/s; 154/1614 objects degraded (9.542%) 2024-09-17T09:45:37.515 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:45:37 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2[73492]: 2024-09-17T09:45:37.416+0000 7f12467e9640 -1 osd.2 103 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T09:45:37.839 INFO:teuthology.orchestra.run.smithi078.stdout: 5 225701 10.43 MB/sec execute 392 sec latency 186.012 ms 2024-09-17T09:45:38.023 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:37 smithi078 ceph-mon[63787]: from='osd.2 [v2:172.21.15.78:6818/2245003569,v1:172.21.15.78:6819/2245003569]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-17T09:45:38.024 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:37 smithi078 ceph-mon[63787]: osdmap e106: 8 total, 7 up, 8 in 2024-09-17T09:45:38.024 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:37 smithi078 ceph-mon[63787]: from='osd.2 [v2:172.21.15.78:6818/2245003569,v1:172.21.15.78:6819/2245003569]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]} : dispatch 2024-09-17T09:45:38.024 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:37 smithi078 ceph-mon[63787]: from='osd.2 [v2:172.21.15.78:6818/2245003569,v1:172.21.15.78:6819/2245003569]' entity='osd.2' 2024-09-17T09:45:38.024 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:37 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 154/1614 objects degraded (9.542%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T09:45:38.024 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:37 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:45:38.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:37 smithi136 ceph-mon[49560]: from='osd.2 [v2:172.21.15.78:6818/2245003569,v1:172.21.15.78:6819/2245003569]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-17T09:45:38.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:37 smithi136 ceph-mon[49560]: osdmap e106: 8 total, 7 up, 8 in 2024-09-17T09:45:38.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:37 smithi136 ceph-mon[49560]: from='osd.2 [v2:172.21.15.78:6818/2245003569,v1:172.21.15.78:6819/2245003569]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]} : dispatch 2024-09-17T09:45:38.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:37 smithi136 ceph-mon[49560]: from='osd.2 [v2:172.21.15.78:6818/2245003569,v1:172.21.15.78:6819/2245003569]' entity='osd.2' 2024-09-17T09:45:38.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:37 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 154/1614 objects degraded (9.542%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T09:45:38.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:37 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:45:38.839 INFO:teuthology.orchestra.run.smithi078.stdout: 5 226015 10.43 MB/sec execute 393 sec latency 450.825 ms 2024-09-17T09:45:39.048 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:38 smithi078 ceph-mon[63787]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:45:39.049 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:38 smithi078 ceph-mon[63787]: osd.2 [v2:172.21.15.78:6818/2245003569,v1:172.21.15.78:6819/2245003569] boot 2024-09-17T09:45:39.049 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:38 smithi078 ceph-mon[63787]: osdmap e107: 8 total, 8 up, 8 in 2024-09-17T09:45:39.049 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-17T09:45:39.049 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:38 smithi078 ceph-mon[63787]: pgmap v150: 97 pgs: 3 peering, 5 active+undersized, 17 active+undersized+degraded, 72 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 3.9 MiB/s wr, 418 op/s; 115/1581 objects degraded (7.274%) 2024-09-17T09:45:39.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:38 smithi136 ceph-mon[49560]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:45:39.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:38 smithi136 ceph-mon[49560]: osd.2 [v2:172.21.15.78:6818/2245003569,v1:172.21.15.78:6819/2245003569] boot 2024-09-17T09:45:39.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:38 smithi136 ceph-mon[49560]: osdmap e107: 8 total, 8 up, 8 in 2024-09-17T09:45:39.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-17T09:45:39.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:38 smithi136 ceph-mon[49560]: pgmap v150: 97 pgs: 3 peering, 5 active+undersized, 17 active+undersized+degraded, 72 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 3.9 MiB/s wr, 418 op/s; 115/1581 objects degraded (7.274%) 2024-09-17T09:45:39.840 INFO:teuthology.orchestra.run.smithi078.stdout: 5 226121 10.41 MB/sec execute 394 sec latency 1452.283 ms 2024-09-17T09:45:40.465 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:40 smithi078 ceph-mon[63787]: osdmap e108: 8 total, 8 up, 8 in 2024-09-17T09:45:40.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:40 smithi136 ceph-mon[49560]: osdmap e108: 8 total, 8 up, 8 in 2024-09-17T09:45:40.840 INFO:teuthology.orchestra.run.smithi078.stdout: 5 226243 10.41 MB/sec execute 395 sec latency 2086.420 ms 2024-09-17T09:45:41.477 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:41 smithi078 ceph-mon[63787]: osdmap e109: 8 total, 8 up, 8 in 2024-09-17T09:45:41.478 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:41 smithi078 ceph-mon[63787]: pgmap v153: 97 pgs: 16 peering, 3 active+undersized, 6 active+undersized+degraded, 72 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.4 MiB/s wr, 729 op/s; 46/1626 objects degraded (2.829%) 2024-09-17T09:45:41.478 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:41 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:41.478 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:41 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:41.478 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:41 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:45:41.478 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:41 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:45:41.478 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:41 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:41.478 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:41 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:41.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:41 smithi136 ceph-mon[49560]: osdmap e109: 8 total, 8 up, 8 in 2024-09-17T09:45:41.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:41 smithi136 ceph-mon[49560]: pgmap v153: 97 pgs: 16 peering, 3 active+undersized, 6 active+undersized+degraded, 72 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.4 MiB/s wr, 729 op/s; 46/1626 objects degraded (2.829%) 2024-09-17T09:45:41.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:41 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:41.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:41 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:41.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:41 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:45:41.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:41 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:45:41.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:41 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:41.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:41 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:41.840 INFO:teuthology.orchestra.run.smithi078.stdout: 5 226576 10.41 MB/sec execute 396 sec latency 389.605 ms 2024-09-17T09:45:42.496 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:42 smithi078 ceph-mon[63787]: osdmap e110: 8 total, 8 up, 8 in 2024-09-17T09:45:42.497 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:42 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:45:42.497 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:42 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:45:42.497 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:42 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:45:42.497 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:42 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:45:42.497 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:42 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-17T09:45:42.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:42 smithi136 ceph-mon[49560]: osdmap e110: 8 total, 8 up, 8 in 2024-09-17T09:45:42.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:42 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:45:42.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:42 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:45:42.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:42 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:45:42.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:42 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:45:42.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:42 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-17T09:45:42.845 INFO:teuthology.orchestra.run.smithi078.stdout: 5 226809 10.40 MB/sec execute 397 sec latency 996.217 ms 2024-09-17T09:45:43.509 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:43 smithi078 ceph-mon[63787]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T09:45:43.510 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:43 smithi078 ceph-mon[63787]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-09-17T09:45:43.510 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:43 smithi078 ceph-mon[63787]: osdmap e111: 8 total, 8 up, 8 in 2024-09-17T09:45:43.510 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:43 smithi078 ceph-mon[63787]: pgmap v156: 97 pgs: 1 active+undersized+remapped, 17 peering, 79 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 5.1 MiB/s wr, 490 op/s; 321 KiB/s, 13 objects/s recovering 2024-09-17T09:45:43.510 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:43 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 46/1626 objects degraded (2.829%), 6 pgs degraded (PG_DEGRADED) 2024-09-17T09:45:43.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:43 smithi136 ceph-mon[49560]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T09:45:43.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:43 smithi136 ceph-mon[49560]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-09-17T09:45:43.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:43 smithi136 ceph-mon[49560]: osdmap e111: 8 total, 8 up, 8 in 2024-09-17T09:45:43.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:43 smithi136 ceph-mon[49560]: pgmap v156: 97 pgs: 1 active+undersized+remapped, 17 peering, 79 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 5.1 MiB/s wr, 490 op/s; 321 KiB/s, 13 objects/s recovering 2024-09-17T09:45:43.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:43 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 46/1626 objects degraded (2.829%), 6 pgs degraded (PG_DEGRADED) 2024-09-17T09:45:43.845 INFO:teuthology.orchestra.run.smithi078.stdout: 5 227085 10.39 MB/sec execute 398 sec latency 788.738 ms 2024-09-17T09:45:44.524 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:44 smithi078 ceph-mon[63787]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 46/1626 objects degraded (2.829%), 6 pgs degraded) 2024-09-17T09:45:44.524 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:44 smithi078 ceph-mon[63787]: Cluster is now healthy 2024-09-17T09:45:44.525 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:44 smithi078 ceph-mon[63787]: osdmap e112: 8 total, 8 up, 8 in 2024-09-17T09:45:44.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:44 smithi136 ceph-mon[49560]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 46/1626 objects degraded (2.829%), 6 pgs degraded) 2024-09-17T09:45:44.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:44 smithi136 ceph-mon[49560]: Cluster is now healthy 2024-09-17T09:45:44.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:44 smithi136 ceph-mon[49560]: osdmap e112: 8 total, 8 up, 8 in 2024-09-17T09:45:44.845 INFO:teuthology.orchestra.run.smithi078.stdout: 5 227370 10.38 MB/sec execute 399 sec latency 601.757 ms 2024-09-17T09:45:45.534 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:45 smithi078 ceph-mon[63787]: pgmap v158: 97 pgs: 2 active+undersized+remapped, 2 peering, 93 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 3.9 MiB/s wr, 299 op/s; 1.6 MiB/s, 53 objects/s recovering 2024-09-17T09:45:45.534 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:45 smithi078 ceph-mon[63787]: osdmap e113: 8 total, 8 up, 8 in 2024-09-17T09:45:45.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:45 smithi136 ceph-mon[49560]: pgmap v158: 97 pgs: 2 active+undersized+remapped, 2 peering, 93 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 3.9 MiB/s wr, 299 op/s; 1.6 MiB/s, 53 objects/s recovering 2024-09-17T09:45:45.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:45 smithi136 ceph-mon[49560]: osdmap e113: 8 total, 8 up, 8 in 2024-09-17T09:45:45.845 INFO:teuthology.orchestra.run.smithi078.stdout: 5 227597 10.37 MB/sec execute 400 sec latency 89.238 ms 2024-09-17T09:45:46.845 INFO:teuthology.orchestra.run.smithi078.stdout: 5 227848 10.35 MB/sec execute 401 sec latency 118.138 ms 2024-09-17T09:45:47.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:47 smithi078 ceph-mon[63787]: pgmap v160: 97 pgs: 2 active+undersized+remapped, 2 peering, 93 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 3.1 MiB/s wr, 240 op/s; 1.3 MiB/s, 43 objects/s recovering 2024-09-17T09:45:47.728 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:47 smithi136 ceph-mon[49560]: pgmap v160: 97 pgs: 2 active+undersized+remapped, 2 peering, 93 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 3.1 MiB/s wr, 240 op/s; 1.3 MiB/s, 43 objects/s recovering 2024-09-17T09:45:47.845 INFO:teuthology.orchestra.run.smithi078.stdout: 5 228125 10.34 MB/sec execute 402 sec latency 73.359 ms 2024-09-17T09:45:48.845 INFO:teuthology.orchestra.run.smithi078.stdout: 5 228357 10.33 MB/sec execute 403 sec latency 91.215 ms 2024-09-17T09:45:49.683 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:49 smithi078 ceph-mon[63787]: pgmap v161: 97 pgs: 1 peering, 96 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.0 MiB/s wr, 287 op/s; 862 KiB/s, 27 objects/s recovering 2024-09-17T09:45:49.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:49 smithi136 ceph-mon[49560]: pgmap v161: 97 pgs: 1 peering, 96 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.0 MiB/s wr, 287 op/s; 862 KiB/s, 27 objects/s recovering 2024-09-17T09:45:49.845 INFO:teuthology.orchestra.run.smithi078.stdout: 5 228630 10.33 MB/sec execute 404 sec latency 44.678 ms 2024-09-17T09:45:50.845 INFO:teuthology.orchestra.run.smithi078.stdout: 5 228993 10.34 MB/sec execute 405 sec latency 48.342 ms 2024-09-17T09:45:51.468 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:51 smithi078 ceph-mon[63787]: pgmap v162: 97 pgs: 97 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 4.0 MiB/s wr, 391 op/s; 647 KiB/s, 20 objects/s recovering 2024-09-17T09:45:51.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:51 smithi136 ceph-mon[49560]: pgmap v162: 97 pgs: 97 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 4.0 MiB/s wr, 391 op/s; 647 KiB/s, 20 objects/s recovering 2024-09-17T09:45:51.845 INFO:teuthology.orchestra.run.smithi078.stdout: 5 229448 10.33 MB/sec execute 406 sec latency 42.793 ms 2024-09-17T09:45:52.845 INFO:teuthology.orchestra.run.smithi078.stdout: 5 229702 10.33 MB/sec execute 407 sec latency 43.787 ms 2024-09-17T09:45:53.513 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:53 smithi078 ceph-mon[63787]: pgmap v163: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 3.7 MiB/s wr, 353 op/s; 575 KiB/s, 12 objects/s recovering 2024-09-17T09:45:53.514 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:53 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:53.514 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:53 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:45:53.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:53 smithi136 ceph-mon[49560]: pgmap v163: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 3.7 MiB/s wr, 353 op/s; 575 KiB/s, 12 objects/s recovering 2024-09-17T09:45:53.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:53 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:53.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:53 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:45:53.845 INFO:teuthology.orchestra.run.smithi078.stdout: 5 229980 10.32 MB/sec execute 408 sec latency 63.558 ms 2024-09-17T09:45:54.845 INFO:teuthology.orchestra.run.smithi078.stdout: 5 230249 10.31 MB/sec execute 409 sec latency 64.238 ms 2024-09-17T09:45:55.590 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:55 smithi078 ceph-mon[63787]: pgmap v164: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.5 MiB/s wr, 449 op/s 2024-09-17T09:45:55.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:55 smithi136 ceph-mon[49560]: pgmap v164: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.5 MiB/s wr, 449 op/s 2024-09-17T09:45:55.845 INFO:teuthology.orchestra.run.smithi078.stdout: 5 230523 10.31 MB/sec execute 410 sec latency 64.749 ms 2024-09-17T09:45:56.845 INFO:teuthology.orchestra.run.smithi078.stdout: 5 230804 10.30 MB/sec execute 411 sec latency 56.514 ms 2024-09-17T09:45:57.642 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:57 smithi078 ceph-mon[63787]: pgmap v165: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.7 MiB/s wr, 374 op/s 2024-09-17T09:45:57.643 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:57 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-17T09:45:57.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:57 smithi136 ceph-mon[49560]: pgmap v165: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.7 MiB/s wr, 374 op/s 2024-09-17T09:45:57.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:57 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-17T09:45:57.845 INFO:teuthology.orchestra.run.smithi078.stdout: 5 231075 10.29 MB/sec execute 412 sec latency 52.262 ms 2024-09-17T09:45:58.594 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-17T09:45:58.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:58 smithi078 ceph-mon[63787]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T09:45:58.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:58 smithi078 ceph-mon[63787]: Upgrade: osd.3 is safe to restart 2024-09-17T09:45:58.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:58 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:58.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:58 smithi136 ceph-mon[49560]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T09:45:58.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:58 smithi136 ceph-mon[49560]: Upgrade: osd.3 is safe to restart 2024-09-17T09:45:58.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:58 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:58.845 INFO:teuthology.orchestra.run.smithi078.stdout: 5 231461 10.29 MB/sec execute 413 sec latency 42.656 ms 2024-09-17T09:45:59.624 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:45:59.625 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (6m) 24s ago 15m 17.6M - 0.27.0 11f11916f8cd 97db2373ce92 2024-09-17T09:45:59.625 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (2m) 24s ago 15m 6899k - 19.3.0-4963-ge646f729 f271290a0950 da528c5ca445 2024-09-17T09:45:59.625 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi136 smithi136 running (2m) 2m ago 14m 6899k - 19.3.0-4963-ge646f729 f271290a0950 909f405348cc 2024-09-17T09:45:59.625 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (6m) 24s ago 15m 53.4M - dad864ee21e9 db74b0f8e0a7 2024-09-17T09:45:59.625 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.jdrsup smithi078 running (9m) 24s ago 9m 38.2M - 17.2.0 e1d6a67b021e 8f834e15ba88 2024-09-17T09:45:59.625 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi136.smslff smithi136 running (9m) 2m ago 9m 15.8M - 17.2.0 e1d6a67b021e 0c20e2b328d3 2024-09-17T09:45:59.625 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rsqofi smithi078 *:8443,9283,8765 running (5m) 24s ago 16m 517M - 19.3.0-4963-ge646f729 f271290a0950 7dda3108c9a0 2024-09-17T09:45:59.625 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi136.hicmlm smithi136 *:8443,9283,8765 running (4m) 2m ago 14m 453M - 19.3.0-4963-ge646f729 f271290a0950 fc518b58e07a 2024-09-17T09:45:59.625 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (4m) 24s ago 16m 51.2M 2048M 19.3.0-4963-ge646f729 f271290a0950 3ead101d3db1 2024-09-17T09:45:59.625 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi136 smithi136 running (3m) 2m ago 14m 36.0M 2048M 19.3.0-4963-ge646f729 f271290a0950 4fce88842b79 2024-09-17T09:45:59.625 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.tcwbvi smithi078 *:2049 running (9m) 24s ago 9m 282M - 4.0 e1d6a67b021e 30c87cb46555 2024-09-17T09:45:59.626 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi136.regjvc smithi136 *:2049 running (9m) 2m ago 9m 87.7M - 4.0 e1d6a67b021e b88e4dc24376 2024-09-17T09:45:59.626 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (6m) 24s ago 15m 10.9M - 1.7.0 72c9c2088986 52dad3adeeaa 2024-09-17T09:45:59.626 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi136 smithi136 *:9100 running (5m) 2m ago 14m 11.9M - 1.7.0 72c9c2088986 f2bc5856e35a 2024-09-17T09:45:59.626 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (2m) 24s ago 13m 305M 1936M 19.3.0-4963-ge646f729 f271290a0950 1ea81cc809b8 2024-09-17T09:45:59.626 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (83s) 24s ago 13m 236M 1936M 19.3.0-4963-ge646f729 f271290a0950 eda32560334b 2024-09-17T09:45:59.626 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (33s) 24s ago 12m 11.5M 1936M 19.3.0-4963-ge646f729 f271290a0950 83a32ad7c358 2024-09-17T09:45:59.626 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (12m) 24s ago 12m 440M 1936M 17.2.0 e1d6a67b021e 1b3a8be89881 2024-09-17T09:45:59.626 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi136 running (12m) 2m ago 12m 557M 2704M 17.2.0 e1d6a67b021e a0c5b4de2414 2024-09-17T09:45:59.626 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi136 running (11m) 2m ago 11m 383M 2704M 17.2.0 e1d6a67b021e 30a3049955f9 2024-09-17T09:45:59.626 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi136 running (11m) 2m ago 11m 572M 2704M 17.2.0 e1d6a67b021e 1698e1567c33 2024-09-17T09:45:59.626 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi136 running (10m) 2m ago 10m 453M 2704M 17.2.0 e1d6a67b021e 5d3b08cecd78 2024-09-17T09:45:59.626 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (4m) 24s ago 14m 53.8M - 2.51.0 1d3b7f56885b 6e5e8c721eb0 2024-09-17T09:45:59.663 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:59 smithi078 ceph-mon[63787]: pgmap v166: 97 pgs: 97 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.5 MiB/s wr, 494 op/s 2024-09-17T09:45:59.663 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:59 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:59.663 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:59 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-17T09:45:59.663 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:45:59 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:45:59.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:59 smithi136 ceph-mon[49560]: pgmap v166: 97 pgs: 97 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.5 MiB/s wr, 494 op/s 2024-09-17T09:45:59.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:59 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:45:59.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:59 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-17T09:45:59.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:45:59 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:45:59.845 INFO:teuthology.orchestra.run.smithi078.stdout: 5 231801 10.28 MB/sec execute 414 sec latency 47.710 ms 2024-09-17T09:46:00.233 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:46:00.233 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-17T09:46:00.234 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:46:00.234 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:46:00.234 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-17T09:46:00.234 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:46:00.234 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:46:00.234 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-17T09:46:00.234 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-17T09:46:00.234 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 3 2024-09-17T09:46:00.234 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:46:00.234 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-17T09:46:00.234 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:46:00.234 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:46:00.235 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-17T09:46:00.235 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-17T09:46:00.235 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 7 2024-09-17T09:46:00.235 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-17T09:46:00.235 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:46:00.697 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:00 smithi078 ceph-mon[63787]: from='client.34306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:46:00.698 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:00 smithi078 ceph-mon[63787]: from='client.34310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:46:00.698 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:00 smithi078 ceph-mon[63787]: Upgrade: Updating osd.3 2024-09-17T09:46:00.698 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:00 smithi078 ceph-mon[63787]: Deploying daemon osd.3 on smithi078 2024-09-17T09:46:00.698 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:00 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/2352658008' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:46:00.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:00 smithi136 ceph-mon[49560]: from='client.34306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:46:00.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:00 smithi136 ceph-mon[49560]: from='client.34310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:46:00.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:00 smithi136 ceph-mon[49560]: Upgrade: Updating osd.3 2024-09-17T09:46:00.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:00 smithi136 ceph-mon[49560]: Deploying daemon osd.3 on smithi078 2024-09-17T09:46:00.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:00 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/2352658008' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:46:00.777 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:46:00.777 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407", 2024-09-17T09:46:00.777 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-17T09:46:00.777 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T09:46:00.777 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-17T09:46:00.777 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-09-17T09:46:00.778 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-17T09:46:00.778 INFO:teuthology.orchestra.run.smithi078.stdout: "mon" 2024-09-17T09:46:00.778 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-17T09:46:00.778 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "9/23 daemons upgraded", 2024-09-17T09:46:00.778 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T09:46:00.778 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-17T09:46:00.778 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:46:00.845 INFO:teuthology.orchestra.run.smithi078.stdout: 5 232031 10.27 MB/sec execute 415 sec latency 59.413 ms 2024-09-17T09:46:01.448 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-17T09:46:01.713 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:01 smithi078 ceph-mon[63787]: from='client.34314 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:46:01.713 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:01 smithi078 ceph-mon[63787]: pgmap v167: 97 pgs: 97 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.8 MiB/s wr, 460 op/s 2024-09-17T09:46:01.713 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:01 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/816826715' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:46:01.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:01 smithi136 ceph-mon[49560]: from='client.34314 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:46:01.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:01 smithi136 ceph-mon[49560]: pgmap v167: 97 pgs: 97 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.8 MiB/s wr, 460 op/s 2024-09-17T09:46:01.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:01 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/816826715' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:46:01.845 INFO:teuthology.orchestra.run.smithi078.stdout: 5 232382 10.29 MB/sec execute 416 sec latency 47.181 ms 2024-09-17T09:46:02.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:02 smithi136 ceph-mon[49560]: from='client.44247 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:46:02.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:02 smithi078 ceph-mon[63787]: from='client.44247 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:46:02.845 INFO:teuthology.orchestra.run.smithi078.stdout: 5 232743 10.28 MB/sec execute 417 sec latency 57.499 ms 2024-09-17T09:46:03.522 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:03 smithi078 ceph-mon[63787]: pgmap v168: 97 pgs: 97 active+clean; 346 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.4 MiB/s wr, 479 op/s 2024-09-17T09:46:03.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:03 smithi136 ceph-mon[49560]: pgmap v168: 97 pgs: 97 active+clean; 346 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.4 MiB/s wr, 479 op/s 2024-09-17T09:46:03.845 INFO:teuthology.orchestra.run.smithi078.stdout: 5 233070 10.27 MB/sec execute 418 sec latency 62.658 ms 2024-09-17T09:46:04.525 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:04 smithi078 systemd[1]: Stopping Ceph osd.3 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:46:04.798 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:04 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3[45123]: 2024-09-17T09:46:04.759+0000 7fe9540f5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:46:04.798 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:04 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3[45123]: 2024-09-17T09:46:04.759+0000 7fe9540f5700 -1 osd.3 113 *** Got signal Terminated *** 2024-09-17T09:46:04.798 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:04 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3[45123]: 2024-09-17T09:46:04.759+0000 7fe9540f5700 -1 osd.3 113 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:46:04.846 INFO:teuthology.orchestra.run.smithi078.stdout: 5 233286 10.26 MB/sec execute 419 sec latency 66.273 ms 2024-09-17T09:46:05.567 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:05 smithi078 ceph-mon[63787]: pgmap v169: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.5 MiB/s wr, 534 op/s 2024-09-17T09:46:05.567 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:05 smithi078 ceph-mon[63787]: osd.3 marked itself down and dead 2024-09-17T09:46:05.845 INFO:teuthology.orchestra.run.smithi078.stdout: 5 233567 10.26 MB/sec execute 420 sec latency 367.029 ms 2024-09-17T09:46:05.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:05 smithi136 ceph-mon[49560]: pgmap v169: 97 pgs: 97 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.5 MiB/s wr, 534 op/s 2024-09-17T09:46:05.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:05 smithi136 ceph-mon[49560]: osd.3 marked itself down and dead 2024-09-17T09:46:06.325 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:06 smithi078 podman[75434]: 2024-09-17 09:46:06.232667969 +0000 UTC m=+1.646327948 container died 1b3a8be89881d78286d40cc01d881a4933fc5318b6530cf5c0fd651652e1f9e6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, release=754, GIT_BRANCH=HEAD, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, architecture=x86_64, ceph=True) 2024-09-17T09:46:06.578 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:06 smithi078 ceph-mon[63787]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:46:06.578 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:06 smithi078 ceph-mon[63787]: osdmap e114: 8 total, 7 up, 8 in 2024-09-17T09:46:06.578 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:06 smithi078 ceph-mon[63787]: osdmap e115: 8 total, 7 up, 8 in 2024-09-17T09:46:06.846 INFO:teuthology.orchestra.run.smithi078.stdout: 5 233757 10.24 MB/sec execute 421 sec latency 1031.786 ms 2024-09-17T09:46:06.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:06 smithi136 ceph-mon[49560]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:46:06.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:06 smithi136 ceph-mon[49560]: osdmap e114: 8 total, 7 up, 8 in 2024-09-17T09:46:06.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:06 smithi136 ceph-mon[49560]: osdmap e115: 8 total, 7 up, 8 in 2024-09-17T09:46:07.632 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:07 smithi078 ceph-mon[63787]: pgmap v171: 97 pgs: 16 stale+active+clean, 81 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.3 MiB/s wr, 438 op/s 2024-09-17T09:46:07.632 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:07 smithi078 podman[75434]: 2024-09-17 09:46:07.519869786 +0000 UTC m=+2.933529756 container cleanup 1b3a8be89881d78286d40cc01d881a4933fc5318b6530cf5c0fd651652e1f9e6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, version=8, distribution-scope=public, release=754, vcs-type=git, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-09-17T09:46:07.632 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:07 smithi078 bash[75434]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3 2024-09-17T09:46:07.846 INFO:teuthology.orchestra.run.smithi078.stdout: 5 234150 10.25 MB/sec execute 422 sec latency 34.902 ms 2024-09-17T09:46:07.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:07 smithi136 ceph-mon[49560]: pgmap v171: 97 pgs: 16 stale+active+clean, 81 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.3 MiB/s wr, 438 op/s 2024-09-17T09:46:08.155 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:07 smithi078 podman[75449]: 2024-09-17 09:46:07.96801657 +0000 UTC m=+1.730720734 container remove 1b3a8be89881d78286d40cc01d881a4933fc5318b6530cf5c0fd651652e1f9e6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, RELEASE=HEAD, ceph=True, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, name=centos-stream, release=754, vcs-type=git, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, version=8, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0) 2024-09-17T09:46:08.665 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:08 smithi078 podman[75534]: 2024-09-17 09:46:08.439161532 +0000 UTC m=+0.022853218 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:46:08.733 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:08 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:08.733 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:08 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:46:08.733 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:08 smithi078 ceph-mon[63787]: pgmap v173: 97 pgs: 9 active+undersized, 11 stale+active+clean, 9 active+undersized+degraded, 68 active+clean; 342 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.0 MiB/s wr, 383 op/s; 39/1659 objects degraded (2.351%) 2024-09-17T09:46:08.846 INFO:teuthology.orchestra.run.smithi078.stdout: 5 234440 10.24 MB/sec execute 423 sec latency 55.661 ms 2024-09-17T09:46:08.923 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:08 smithi078 podman[75534]: 2024-09-17 09:46:08.760258594 +0000 UTC m=+0.343950261 container create bdd429a96f6fb49c7c8736ec84854569b51c778ebc889f48353221ea18053e76 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72) 2024-09-17T09:46:08.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:08 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:08.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:08 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:46:08.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:08 smithi136 ceph-mon[49560]: pgmap v173: 97 pgs: 9 active+undersized, 11 stale+active+clean, 9 active+undersized+degraded, 68 active+clean; 342 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.0 MiB/s wr, 383 op/s; 39/1659 objects degraded (2.351%) 2024-09-17T09:46:09.184 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:09 smithi078 podman[75534]: 2024-09-17 09:46:09.169796328 +0000 UTC m=+0.753487999 container init bdd429a96f6fb49c7c8736ec84854569b51c778ebc889f48353221ea18053e76 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3-deactivate, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T09:46:09.443 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:09 smithi078 podman[75534]: 2024-09-17 09:46:09.185588477 +0000 UTC m=+0.769280147 container start bdd429a96f6fb49c7c8736ec84854569b51c778ebc889f48353221ea18053e76 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-17T09:46:09.443 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:09 smithi078 podman[75534]: 2024-09-17 09:46:09.362015507 +0000 UTC m=+0.945707177 container attach bdd429a96f6fb49c7c8736ec84854569b51c778ebc889f48353221ea18053e76 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T09:46:09.711 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:09 smithi078 podman[75534]: 2024-09-17 09:46:09.532773002 +0000 UTC m=+1.116464681 container died bdd429a96f6fb49c7c8736ec84854569b51c778ebc889f48353221ea18053e76 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:46:09.846 INFO:teuthology.orchestra.run.smithi078.stdout: 5 234731 10.24 MB/sec execute 424 sec latency 49.337 ms 2024-09-17T09:46:09.970 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:09 smithi078 ceph-mon[63787]: Health check failed: Degraded data redundancy: 39/1659 objects degraded (2.351%), 9 pgs degraded (PG_DEGRADED) 2024-09-17T09:46:10.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:09 smithi136 ceph-mon[49560]: Health check failed: Degraded data redundancy: 39/1659 objects degraded (2.351%), 9 pgs degraded (PG_DEGRADED) 2024-09-17T09:46:10.727 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:10 smithi078 podman[75557]: 2024-09-17 09:46:10.59522508 +0000 UTC m=+1.045069596 container remove bdd429a96f6fb49c7c8736ec84854569b51c778ebc889f48353221ea18053e76 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T09:46:10.727 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:10 smithi078 systemd[1]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.3.service: Deactivated successfully. 2024-09-17T09:46:10.727 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:10 smithi078 systemd[1]: Stopped Ceph osd.3 for 35c0cd22-74d7-11ef-bceb-c7b262605968. 2024-09-17T09:46:10.727 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:10 smithi078 systemd[1]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.3.service: Consumed 4min 38.554s CPU time. 2024-09-17T09:46:10.846 INFO:teuthology.orchestra.run.smithi078.stdout: 5 235087 10.22 MB/sec execute 425 sec latency 80.197 ms 2024-09-17T09:46:10.982 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:10 smithi078 ceph-mon[63787]: pgmap v174: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 2.2 KiB/s rd, 5.0 MiB/s wr, 402 op/s; 194/1572 objects degraded (12.341%) 2024-09-17T09:46:11.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:10 smithi136 ceph-mon[49560]: pgmap v174: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 2.2 KiB/s rd, 5.0 MiB/s wr, 402 op/s; 194/1572 objects degraded (12.341%) 2024-09-17T09:46:11.234 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:10 smithi078 systemd[1]: Starting Ceph osd.3 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:46:11.491 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:11 smithi078 podman[75654]: 2024-09-17 09:46:11.155508637 +0000 UTC m=+0.021698513 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:46:11.492 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:11 smithi078 podman[75654]: 2024-09-17 09:46:11.420184541 +0000 UTC m=+0.286374402 container create 5cb1948b670c51ecc6e40ae2330ce0371b7b35d02463740c7a2b40bc901adf8d (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3-activate, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-17T09:46:11.846 INFO:teuthology.orchestra.run.smithi078.stdout: 5 235420 10.22 MB/sec execute 426 sec latency 43.255 ms 2024-09-17T09:46:12.020 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:11 smithi078 podman[75654]: 2024-09-17 09:46:11.948771222 +0000 UTC m=+0.814961106 container init 5cb1948b670c51ecc6e40ae2330ce0371b7b35d02463740c7a2b40bc901adf8d (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-17T09:46:12.020 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:11 smithi078 podman[75654]: 2024-09-17 09:46:11.958729466 +0000 UTC m=+0.824919328 container start 5cb1948b670c51ecc6e40ae2330ce0371b7b35d02463740c7a2b40bc901adf8d (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3-activate, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-17T09:46:12.181 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:12 smithi078 podman[75654]: 2024-09-17 09:46:12.143006546 +0000 UTC m=+1.009196414 container attach 5cb1948b670c51ecc6e40ae2330ce0371b7b35d02463740c7a2b40bc901adf8d (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.build-date=20240731, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-17T09:46:12.181 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:12 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3-activate[75668]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:46:12.181 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:12 smithi078 bash[75654]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:46:12.181 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:12 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3-activate[75668]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:46:12.181 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:12 smithi078 bash[75654]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:46:12.846 INFO:teuthology.orchestra.run.smithi078.stdout: 5 235814 10.23 MB/sec execute 427 sec latency 51.438 ms 2024-09-17T09:46:12.935 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:12 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3-activate[75668]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-17T09:46:12.935 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:12 smithi078 bash[75654]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-17T09:46:12.935 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:12 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3-activate[75668]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-17T09:46:12.935 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:12 smithi078 bash[75654]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-17T09:46:13.189 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:13 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3-activate[75668]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-17T09:46:13.189 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:13 smithi078 bash[75654]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-17T09:46:13.189 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:13 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3-activate[75668]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-17T09:46:13.189 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:13 smithi078 bash[75654]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-17T09:46:13.189 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:13 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3-activate[75668]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-17T09:46:13.189 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:13 smithi078 bash[75654]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-17T09:46:13.189 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:13 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3-activate[75668]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-17T09:46:13.189 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:13 smithi078 bash[75654]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-17T09:46:13.189 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:13 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3-activate[75668]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-17T09:46:13.190 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:13 smithi078 bash[75654]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-17T09:46:13.190 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:13 smithi078 podman[75654]: 2024-09-17 09:46:13.137439299 +0000 UTC m=+2.003629165 container died 5cb1948b670c51ecc6e40ae2330ce0371b7b35d02463740c7a2b40bc901adf8d (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-17T09:46:13.691 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:13 smithi078 ceph-mon[63787]: pgmap v175: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.4 MiB/s wr, 329 op/s; 199/1572 objects degraded (12.659%) 2024-09-17T09:46:13.846 INFO:teuthology.orchestra.run.smithi078.stdout: 5 236240 10.24 MB/sec execute 428 sec latency 65.917 ms 2024-09-17T09:46:13.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:13 smithi136 ceph-mon[49560]: pgmap v175: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.4 MiB/s wr, 329 op/s; 199/1572 objects degraded (12.659%) 2024-09-17T09:46:14.458 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:14 smithi078 podman[75798]: 2024-09-17 09:46:14.417644115 +0000 UTC m=+1.268454119 container remove 5cb1948b670c51ecc6e40ae2330ce0371b7b35d02463740c7a2b40bc901adf8d (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T09:46:14.709 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:14 smithi078 ceph-mon[63787]: pgmap v176: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.5 MiB/s wr, 570 op/s; 212/1638 objects degraded (12.943%) 2024-09-17T09:46:14.709 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:14 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 212/1638 objects degraded (12.943%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T09:46:14.710 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:14 smithi078 podman[75855]: 2024-09-17 09:46:14.571221395 +0000 UTC m=+0.021968389 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:46:14.710 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:14 smithi078 podman[75855]: 2024-09-17 09:46:14.678756828 +0000 UTC m=+0.129503812 container create e2cd18998d2b4d67879058627c83bbd4cd393d9f4692e7de722d49815708f82e (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True) 2024-09-17T09:46:14.846 INFO:teuthology.orchestra.run.smithi078.stdout: 5 236625 10.23 MB/sec execute 429 sec latency 57.935 ms 2024-09-17T09:46:14.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:14 smithi136 ceph-mon[49560]: pgmap v176: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.5 MiB/s wr, 570 op/s; 212/1638 objects degraded (12.943%) 2024-09-17T09:46:14.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:14 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 212/1638 objects degraded (12.943%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T09:46:15.228 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:15 smithi078 podman[75855]: 2024-09-17 09:46:15.086221194 +0000 UTC m=+0.536968181 container init e2cd18998d2b4d67879058627c83bbd4cd393d9f4692e7de722d49815708f82e (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3, org.label-schema.build-date=20240731, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:46:15.229 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:15 smithi078 podman[75855]: 2024-09-17 09:46:15.099404167 +0000 UTC m=+0.550151153 container start e2cd18998d2b4d67879058627c83bbd4cd393d9f4692e7de722d49815708f82e (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731) 2024-09-17T09:46:15.229 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:15 smithi078 ceph-osd[75872]: -- 172.21.15.78:0/3089042532 <== mon.0 v2:172.21.15.78:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x561014bae1e0 con 0x561013db9c00 2024-09-17T09:46:15.479 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:15 smithi078 bash[75855]: e2cd18998d2b4d67879058627c83bbd4cd393d9f4692e7de722d49815708f82e 2024-09-17T09:46:15.479 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:15 smithi078 systemd[1]: Started Ceph osd.3 for 35c0cd22-74d7-11ef-bceb-c7b262605968. 2024-09-17T09:46:15.846 INFO:teuthology.orchestra.run.smithi078.stdout: 5 236869 10.22 MB/sec execute 430 sec latency 38.590 ms 2024-09-17T09:46:16.500 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:16 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:16.501 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:16 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:16.501 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:16 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:46:16.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:16 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:16.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:16 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:16.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:16 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:46:16.846 INFO:teuthology.orchestra.run.smithi078.stdout: 5 237300 10.23 MB/sec execute 431 sec latency 36.424 ms 2024-09-17T09:46:17.516 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:17 smithi078 ceph-mon[63787]: pgmap v177: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.9 MiB/s wr, 510 op/s; 212/1638 objects degraded (12.943%) 2024-09-17T09:46:17.516 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:17 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3[75868]: 2024-09-17T09:46:17.495+0000 7f7a9595b640 -1 Falling back to public interface 2024-09-17T09:46:17.626 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:17 smithi136 ceph-mon[49560]: pgmap v177: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.9 MiB/s wr, 510 op/s; 212/1638 objects degraded (12.943%) 2024-09-17T09:46:17.846 INFO:teuthology.orchestra.run.smithi078.stdout: 5 237668 10.23 MB/sec execute 432 sec latency 46.143 ms 2024-09-17T09:46:18.846 INFO:teuthology.orchestra.run.smithi078.stdout: 5 238003 10.23 MB/sec execute 433 sec latency 36.775 ms 2024-09-17T09:46:19.547 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:19 smithi078 ceph-mon[63787]: pgmap v178: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.2 MiB/s wr, 536 op/s; 208/1608 objects degraded (12.935%) 2024-09-17T09:46:19.846 INFO:teuthology.orchestra.run.smithi078.stdout: 5 238314 10.22 MB/sec execute 434 sec latency 46.139 ms 2024-09-17T09:46:19.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:19 smithi136 ceph-mon[49560]: pgmap v178: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.2 MiB/s wr, 536 op/s; 208/1608 objects degraded (12.935%) 2024-09-17T09:46:20.555 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:20 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 216/1632 objects degraded (13.235%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T09:46:20.846 INFO:teuthology.orchestra.run.smithi078.stdout: 5 238723 10.21 MB/sec execute 435 sec latency 34.514 ms 2024-09-17T09:46:20.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:20 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 216/1632 objects degraded (13.235%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T09:46:21.593 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:21 smithi078 ceph-mon[63787]: pgmap v179: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.6 MiB/s wr, 550 op/s; 216/1632 objects degraded (13.235%) 2024-09-17T09:46:21.846 INFO:teuthology.orchestra.run.smithi078.stdout: 5 239061 10.21 MB/sec execute 436 sec latency 33.840 ms 2024-09-17T09:46:21.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:21 smithi136 ceph-mon[49560]: pgmap v179: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.6 MiB/s wr, 550 op/s; 216/1632 objects degraded (13.235%) 2024-09-17T09:46:22.846 INFO:teuthology.orchestra.run.smithi078.stdout: 5 239530 10.23 MB/sec execute 437 sec latency 51.682 ms 2024-09-17T09:46:23.636 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:23 smithi078 ceph-mon[63787]: pgmap v180: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.5 KiB/s rd, 4.8 MiB/s wr, 505 op/s; 213/1623 objects degraded (13.124%) 2024-09-17T09:46:23.636 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:23 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:23.637 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:23 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:46:23.846 INFO:teuthology.orchestra.run.smithi078.stdout: 5 239929 10.23 MB/sec execute 438 sec latency 49.972 ms 2024-09-17T09:46:23.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:23 smithi136 ceph-mon[49560]: pgmap v180: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.5 KiB/s rd, 4.8 MiB/s wr, 505 op/s; 213/1623 objects degraded (13.124%) 2024-09-17T09:46:23.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:23 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:23.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:23 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:46:24.846 INFO:teuthology.orchestra.run.smithi078.stdout: 5 240247 10.23 MB/sec execute 439 sec latency 52.713 ms 2024-09-17T09:46:25.422 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:25 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3[75868]: 2024-09-17T09:46:25.190+0000 7f7a9595b640 -1 osd.3 0 read_superblock omap replica is missing. 2024-09-17T09:46:25.684 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:25 smithi078 ceph-mon[63787]: pgmap v181: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 349 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.3 MiB/s wr, 652 op/s; 217/1635 objects degraded (13.272%) 2024-09-17T09:46:25.846 INFO:teuthology.orchestra.run.smithi078.stdout: 5 240486 10.21 MB/sec execute 440 sec latency 53.882 ms 2024-09-17T09:46:25.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:25 smithi136 ceph-mon[49560]: pgmap v181: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 349 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.3 MiB/s wr, 652 op/s; 217/1635 objects degraded (13.272%) 2024-09-17T09:46:26.725 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:26 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3[75868]: 2024-09-17T09:46:26.651+0000 7f7a9595b640 -1 osd.3 113 log_to_monitors true 2024-09-17T09:46:26.846 INFO:teuthology.orchestra.run.smithi078.stdout: 5 240799 10.21 MB/sec execute 441 sec latency 53.822 ms 2024-09-17T09:46:27.525 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:27 smithi078 ceph-mon[63787]: pgmap v182: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 349 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 447 op/s; 217/1635 objects degraded (13.272%) 2024-09-17T09:46:27.526 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:27 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:27.526 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:27 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:27.526 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:27 smithi078 ceph-mon[63787]: from='osd.3 [v2:172.21.15.78:6826/3004770443,v1:172.21.15.78:6827/3004770443]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-17T09:46:27.526 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:27 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:27.526 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:27 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:27.526 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:27 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 217/1635 objects degraded (13.272%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T09:46:27.846 INFO:teuthology.orchestra.run.smithi078.stdout: 5 241045 10.20 MB/sec execute 442 sec latency 82.379 ms 2024-09-17T09:46:27.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:27 smithi136 ceph-mon[49560]: pgmap v182: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 349 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 447 op/s; 217/1635 objects degraded (13.272%) 2024-09-17T09:46:27.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:27 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:27.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:27 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:27.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:27 smithi136 ceph-mon[49560]: from='osd.3 [v2:172.21.15.78:6826/3004770443,v1:172.21.15.78:6827/3004770443]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-17T09:46:27.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:27 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:27.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:27 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:27.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:27 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 217/1635 objects degraded (13.272%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T09:46:28.281 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:46:28 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3[75868]: 2024-09-17T09:46:28.091+0000 7f7a896fd640 -1 osd.3 113 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T09:46:28.813 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:28 smithi078 ceph-mon[63787]: from='osd.3 [v2:172.21.15.78:6826/3004770443,v1:172.21.15.78:6827/3004770443]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-17T09:46:28.813 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:28 smithi078 ceph-mon[63787]: osdmap e116: 8 total, 7 up, 8 in 2024-09-17T09:46:28.813 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:28 smithi078 ceph-mon[63787]: from='osd.3 [v2:172.21.15.78:6826/3004770443,v1:172.21.15.78:6827/3004770443]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]} : dispatch 2024-09-17T09:46:28.813 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:28 smithi078 ceph-mon[63787]: from='osd.3 [v2:172.21.15.78:6826/3004770443,v1:172.21.15.78:6827/3004770443]' entity='osd.3' 2024-09-17T09:46:28.813 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:28 smithi078 ceph-mon[63787]: pgmap v184: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.1 MiB/s wr, 524 op/s; 218/1617 objects degraded (13.482%) 2024-09-17T09:46:28.846 INFO:teuthology.orchestra.run.smithi078.stdout: 5 241278 10.20 MB/sec execute 443 sec latency 80.527 ms 2024-09-17T09:46:28.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:28 smithi136 ceph-mon[49560]: from='osd.3 [v2:172.21.15.78:6826/3004770443,v1:172.21.15.78:6827/3004770443]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-17T09:46:28.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:28 smithi136 ceph-mon[49560]: osdmap e116: 8 total, 7 up, 8 in 2024-09-17T09:46:28.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:28 smithi136 ceph-mon[49560]: from='osd.3 [v2:172.21.15.78:6826/3004770443,v1:172.21.15.78:6827/3004770443]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]} : dispatch 2024-09-17T09:46:28.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:28 smithi136 ceph-mon[49560]: from='osd.3 [v2:172.21.15.78:6826/3004770443,v1:172.21.15.78:6827/3004770443]' entity='osd.3' 2024-09-17T09:46:28.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:28 smithi136 ceph-mon[49560]: pgmap v184: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.1 MiB/s wr, 524 op/s; 218/1617 objects degraded (13.482%) 2024-09-17T09:46:29.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:29 smithi078 ceph-mon[63787]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:46:29.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:29 smithi078 ceph-mon[63787]: osd.3 [v2:172.21.15.78:6826/3004770443,v1:172.21.15.78:6827/3004770443] boot 2024-09-17T09:46:29.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:29 smithi078 ceph-mon[63787]: osdmap e117: 8 total, 8 up, 8 in 2024-09-17T09:46:29.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:29 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-17T09:46:29.847 INFO:teuthology.orchestra.run.smithi078.stdout: 5 241340 10.18 MB/sec execute 444 sec latency 776.456 ms 2024-09-17T09:46:29.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:29 smithi136 ceph-mon[49560]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:46:29.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:29 smithi136 ceph-mon[49560]: osd.3 [v2:172.21.15.78:6826/3004770443,v1:172.21.15.78:6827/3004770443] boot 2024-09-17T09:46:29.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:29 smithi136 ceph-mon[49560]: osdmap e117: 8 total, 8 up, 8 in 2024-09-17T09:46:29.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:29 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-17T09:46:30.846 INFO:teuthology.orchestra.run.smithi078.stdout: 5 241340 10.16 MB/sec execute 445 sec latency 1776.502 ms 2024-09-17T09:46:31.351 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:31 smithi078 ceph-mon[63787]: osdmap e118: 8 total, 8 up, 8 in 2024-09-17T09:46:31.351 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:31 smithi078 ceph-mon[63787]: pgmap v187: 97 pgs: 6 peering, 12 active+undersized, 23 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 3.5 MiB/s wr, 356 op/s; 169/1545 objects degraded (10.939%) 2024-09-17T09:46:31.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:31 smithi136 ceph-mon[49560]: osdmap e118: 8 total, 8 up, 8 in 2024-09-17T09:46:31.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:31 smithi136 ceph-mon[49560]: pgmap v187: 97 pgs: 6 peering, 12 active+undersized, 23 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 3.5 MiB/s wr, 356 op/s; 169/1545 objects degraded (10.939%) 2024-09-17T09:46:31.848 INFO:teuthology.orchestra.run.smithi078.stdout: 5 241402 10.14 MB/sec execute 446 sec latency 2566.233 ms 2024-09-17T09:46:31.939 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-17T09:46:32.321 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:32 smithi078 ceph-mon[63787]: osdmap e119: 8 total, 8 up, 8 in 2024-09-17T09:46:32.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:32 smithi136 ceph-mon[49560]: osdmap e119: 8 total, 8 up, 8 in 2024-09-17T09:46:32.800 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:46:32.800 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (6m) 6s ago 16m 17.8M - 0.27.0 11f11916f8cd 97db2373ce92 2024-09-17T09:46:32.800 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (3m) 6s ago 15m 6899k - 19.3.0-4963-ge646f729 f271290a0950 da528c5ca445 2024-09-17T09:46:32.800 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi136 smithi136 running (3m) 3m ago 15m 6899k - 19.3.0-4963-ge646f729 f271290a0950 909f405348cc 2024-09-17T09:46:32.800 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (6m) 6s ago 15m 53.0M - dad864ee21e9 db74b0f8e0a7 2024-09-17T09:46:32.800 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.jdrsup smithi078 running (9m) 6s ago 9m 38.4M - 17.2.0 e1d6a67b021e 8f834e15ba88 2024-09-17T09:46:32.800 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi136.smslff smithi136 running (9m) 3m ago 9m 15.8M - 17.2.0 e1d6a67b021e 0c20e2b328d3 2024-09-17T09:46:32.800 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rsqofi smithi078 *:8443,9283,8765 running (6m) 6s ago 17m 519M - 19.3.0-4963-ge646f729 f271290a0950 7dda3108c9a0 2024-09-17T09:46:32.800 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi136.hicmlm smithi136 *:8443,9283,8765 running (5m) 3m ago 15m 453M - 19.3.0-4963-ge646f729 f271290a0950 fc518b58e07a 2024-09-17T09:46:32.800 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (4m) 6s ago 17m 53.2M 2048M 19.3.0-4963-ge646f729 f271290a0950 3ead101d3db1 2024-09-17T09:46:32.801 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi136 smithi136 running (4m) 3m ago 15m 36.0M 2048M 19.3.0-4963-ge646f729 f271290a0950 4fce88842b79 2024-09-17T09:46:32.801 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.tcwbvi smithi078 *:2049 running (9m) 6s ago 9m 284M - 4.0 e1d6a67b021e 30c87cb46555 2024-09-17T09:46:32.801 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi136.regjvc smithi136 *:2049 running (9m) 3m ago 9m 87.7M - 4.0 e1d6a67b021e b88e4dc24376 2024-09-17T09:46:32.801 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (6m) 6s ago 15m 11.0M - 1.7.0 72c9c2088986 52dad3adeeaa 2024-09-17T09:46:32.801 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi136 smithi136 *:9100 running (6m) 3m ago 15m 11.9M - 1.7.0 72c9c2088986 f2bc5856e35a 2024-09-17T09:46:32.801 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (2m) 6s ago 14m 351M 1936M 19.3.0-4963-ge646f729 f271290a0950 1ea81cc809b8 2024-09-17T09:46:32.801 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (116s) 6s ago 13m 290M 1936M 19.3.0-4963-ge646f729 f271290a0950 eda32560334b 2024-09-17T09:46:32.801 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (67s) 6s ago 13m 213M 1936M 19.3.0-4963-ge646f729 f271290a0950 83a32ad7c358 2024-09-17T09:46:32.801 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (18s) 6s ago 13m 14.3M 1936M 19.3.0-4963-ge646f729 f271290a0950 e2cd18998d2b 2024-09-17T09:46:32.801 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi136 running (12m) 3m ago 12m 557M 2704M 17.2.0 e1d6a67b021e a0c5b4de2414 2024-09-17T09:46:32.801 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi136 running (12m) 3m ago 12m 383M 2704M 17.2.0 e1d6a67b021e 30a3049955f9 2024-09-17T09:46:32.801 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi136 running (11m) 3m ago 11m 572M 2704M 17.2.0 e1d6a67b021e 1698e1567c33 2024-09-17T09:46:32.801 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi136 running (11m) 3m ago 11m 453M 2704M 17.2.0 e1d6a67b021e 5d3b08cecd78 2024-09-17T09:46:32.801 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (5m) 6s ago 14m 53.8M - 2.51.0 1d3b7f56885b 6e5e8c721eb0 2024-09-17T09:46:32.847 INFO:teuthology.orchestra.run.smithi078.stdout: 5 241592 10.13 MB/sec execute 447 sec latency 373.317 ms 2024-09-17T09:46:33.112 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:33 smithi078 ceph-mon[63787]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:46:33.113 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:33 smithi078 ceph-mon[63787]: osdmap e120: 8 total, 8 up, 8 in 2024-09-17T09:46:33.113 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:33 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:33.113 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:33 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:33.113 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:33 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:46:33.113 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:33 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:46:33.113 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:33 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:33.113 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:33 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:33.113 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:33 smithi078 ceph-mon[63787]: from='client.34336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:46:33.113 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:33 smithi078 ceph-mon[63787]: pgmap v190: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 7 peering, 9 active+undersized, 16 active+undersized+degraded, 62 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 3.3 MiB/s wr, 249 op/s; 157/1536 objects degraded (10.221%); 2.2 MiB/s, 7 objects/s recovering 2024-09-17T09:46:33.113 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:33 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 169/1545 objects degraded (10.939%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T09:46:33.113 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:33 smithi078 ceph-mon[63787]: osdmap e121: 8 total, 8 up, 8 in 2024-09-17T09:46:33.113 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:33 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:46:33.114 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:33 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:46:33.114 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:33 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:46:33.114 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:33 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:46:33.114 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:33 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-17T09:46:33.291 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:46:33.291 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-17T09:46:33.291 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:46:33.291 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:46:33.291 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-17T09:46:33.291 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:46:33.291 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:46:33.291 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-17T09:46:33.292 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-17T09:46:33.292 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 4 2024-09-17T09:46:33.292 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:46:33.292 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-17T09:46:33.292 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:46:33.292 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:46:33.292 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-17T09:46:33.292 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-17T09:46:33.292 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 8 2024-09-17T09:46:33.292 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-17T09:46:33.292 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:46:33.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:33 smithi136 ceph-mon[49560]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:46:33.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:33 smithi136 ceph-mon[49560]: osdmap e120: 8 total, 8 up, 8 in 2024-09-17T09:46:33.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:33 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:33.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:33 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:33.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:33 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:46:33.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:33 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:46:33.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:33 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:33.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:33 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:33.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:33 smithi136 ceph-mon[49560]: from='client.34336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:46:33.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:33 smithi136 ceph-mon[49560]: pgmap v190: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 7 peering, 9 active+undersized, 16 active+undersized+degraded, 62 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 3.3 MiB/s wr, 249 op/s; 157/1536 objects degraded (10.221%); 2.2 MiB/s, 7 objects/s recovering 2024-09-17T09:46:33.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:33 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 169/1545 objects degraded (10.939%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T09:46:33.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:33 smithi136 ceph-mon[49560]: osdmap e121: 8 total, 8 up, 8 in 2024-09-17T09:46:33.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:33 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:46:33.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:33 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:46:33.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:33 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:46:33.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:33 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:46:33.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:33 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-17T09:46:33.756 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:46:33.756 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407", 2024-09-17T09:46:33.756 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-17T09:46:33.756 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T09:46:33.756 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-17T09:46:33.757 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-09-17T09:46:33.757 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-17T09:46:33.757 INFO:teuthology.orchestra.run.smithi078.stdout: "mon" 2024-09-17T09:46:33.757 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-17T09:46:33.757 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "10/23 daemons upgraded", 2024-09-17T09:46:33.757 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T09:46:33.757 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-17T09:46:33.757 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:46:33.848 INFO:teuthology.orchestra.run.smithi078.stdout: 5 241798 10.11 MB/sec execute 448 sec latency 115.579 ms 2024-09-17T09:46:34.382 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:34 smithi078 ceph-mon[63787]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T09:46:34.383 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:34 smithi078 ceph-mon[63787]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-09-17T09:46:34.383 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:34 smithi078 ceph-mon[63787]: from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:46:34.383 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:34 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/1622957958' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:46:34.383 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:34 smithi078 ceph-mon[63787]: osdmap e122: 8 total, 8 up, 8 in 2024-09-17T09:46:34.404 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_WARN Degraded data redundancy: 157/1536 objects degraded (10.221%), 17 pgs degraded 2024-09-17T09:46:34.405 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 157/1536 objects degraded (10.221%), 17 pgs degraded 2024-09-17T09:46:34.405 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.12 is active+undersized+degraded, acting [5,7] 2024-09-17T09:46:34.405 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.14 is active+undersized+degraded, acting [6,5] 2024-09-17T09:46:34.405 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.19 is active+undersized+degraded, acting [6,0] 2024-09-17T09:46:34.405 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1b is active+undersized+degraded, acting [7,6] 2024-09-17T09:46:34.405 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1e is active+undersized+degraded, acting [0,5] 2024-09-17T09:46:34.405 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.2 is active+undersized+degraded, acting [5,6] 2024-09-17T09:46:34.405 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.5 is active+undersized+degraded, acting [5,2] 2024-09-17T09:46:34.405 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.7 is active+undersized+degraded, acting [7,0] 2024-09-17T09:46:34.405 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.8 is active+undersized+degraded, acting [1,7] 2024-09-17T09:46:34.405 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.b is active+undersized+degraded, acting [0,4] 2024-09-17T09:46:34.405 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.c is active+undersized+degraded, acting [5,6] 2024-09-17T09:46:34.406 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.15 is active+undersized+degraded, acting [7,4] 2024-09-17T09:46:34.406 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.18 is active+undersized+degraded, acting [0,1] 2024-09-17T09:46:34.406 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1e is active+recovery_wait+undersized+degraded+remapped, acting [2,6] 2024-09-17T09:46:34.406 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.0 is active+undersized+degraded, acting [7,0] 2024-09-17T09:46:34.406 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.14 is active+undersized+degraded, acting [2,7] 2024-09-17T09:46:34.406 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.19 is active+undersized+degraded, acting [2,0] 2024-09-17T09:46:34.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:34 smithi136 ceph-mon[49560]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T09:46:34.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:34 smithi136 ceph-mon[49560]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-09-17T09:46:34.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:34 smithi136 ceph-mon[49560]: from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:46:34.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:34 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/1622957958' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:46:34.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:34 smithi136 ceph-mon[49560]: osdmap e122: 8 total, 8 up, 8 in 2024-09-17T09:46:34.848 INFO:teuthology.orchestra.run.smithi078.stdout: 5 242126 10.10 MB/sec execute 449 sec latency 335.824 ms 2024-09-17T09:46:35.502 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:35 smithi078 ceph-mon[63787]: from='client.34348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:46:35.503 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:35 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/1190046950' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:46:35.503 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:35 smithi078 ceph-mon[63787]: pgmap v193: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+remapped, 2 peering, 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 2.2 MiB/s wr, 225 op/s; 26/1575 objects degraded (1.651%); 3.7 MiB/s, 76 objects/s recovering 2024-09-17T09:46:35.503 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:35 smithi078 ceph-mon[63787]: osdmap e123: 8 total, 8 up, 8 in 2024-09-17T09:46:35.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:35 smithi136 ceph-mon[49560]: from='client.34348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:46:35.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:35 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/1190046950' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:46:35.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:35 smithi136 ceph-mon[49560]: pgmap v193: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+remapped, 2 peering, 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 2.2 MiB/s wr, 225 op/s; 26/1575 objects degraded (1.651%); 3.7 MiB/s, 76 objects/s recovering 2024-09-17T09:46:35.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:35 smithi136 ceph-mon[49560]: osdmap e123: 8 total, 8 up, 8 in 2024-09-17T09:46:35.848 INFO:teuthology.orchestra.run.smithi078.stdout: 5 242306 10.09 MB/sec execute 450 sec latency 958.063 ms 2024-09-17T09:46:36.547 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:36 smithi078 ceph-mon[63787]: osdmap e124: 8 total, 8 up, 8 in 2024-09-17T09:46:36.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:36 smithi136 ceph-mon[49560]: osdmap e124: 8 total, 8 up, 8 in 2024-09-17T09:46:36.848 INFO:teuthology.orchestra.run.smithi078.stdout: 5 242590 10.08 MB/sec execute 451 sec latency 58.219 ms 2024-09-17T09:46:37.566 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:37 smithi078 ceph-mon[63787]: pgmap v196: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+remapped, 2 peering, 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.9 MiB/s wr, 213 op/s; 26/1575 objects degraded (1.651%); 1.5 MiB/s, 68 objects/s recovering 2024-09-17T09:46:37.566 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:37 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 26/1575 objects degraded (1.651%), 2 pgs degraded (PG_DEGRADED) 2024-09-17T09:46:37.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:37 smithi136 ceph-mon[49560]: pgmap v196: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+remapped, 2 peering, 1 active+recovering+undersized+degraded+remapped, 88 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.9 MiB/s wr, 213 op/s; 26/1575 objects degraded (1.651%); 1.5 MiB/s, 68 objects/s recovering 2024-09-17T09:46:37.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:37 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 26/1575 objects degraded (1.651%), 2 pgs degraded (PG_DEGRADED) 2024-09-17T09:46:37.848 INFO:teuthology.orchestra.run.smithi078.stdout: 5 242918 10.09 MB/sec execute 452 sec latency 34.759 ms 2024-09-17T09:46:38.831 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:38.831 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:46:38.831 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:38 smithi078 ceph-mon[63787]: pgmap v197: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 94 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2 KiB/s rd, 2.7 MiB/s wr, 333 op/s; 23/1557 objects degraded (1.477%); 1.0 MiB/s, 52 objects/s recovering 2024-09-17T09:46:38.848 INFO:teuthology.orchestra.run.smithi078.stdout: 5 243399 10.10 MB/sec execute 453 sec latency 47.233 ms 2024-09-17T09:46:38.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:38.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:46:38.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:38 smithi136 ceph-mon[49560]: pgmap v197: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 94 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2 KiB/s rd, 2.7 MiB/s wr, 333 op/s; 23/1557 objects degraded (1.477%); 1.0 MiB/s, 52 objects/s recovering 2024-09-17T09:46:39.848 INFO:teuthology.orchestra.run.smithi078.stdout: 5 243785 10.10 MB/sec execute 454 sec latency 40.354 ms 2024-09-17T09:46:40.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:40 smithi078 ceph-mon[63787]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 23/1557 objects degraded (1.477%), 1 pg degraded) 2024-09-17T09:46:40.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:40 smithi078 ceph-mon[63787]: Cluster is now healthy 2024-09-17T09:46:40.848 INFO:teuthology.orchestra.run.smithi078.stdout: 5 244077 10.09 MB/sec execute 455 sec latency 40.389 ms 2024-09-17T09:46:40.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:40 smithi136 ceph-mon[49560]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 23/1557 objects degraded (1.477%), 1 pg degraded) 2024-09-17T09:46:40.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:40 smithi136 ceph-mon[49560]: Cluster is now healthy 2024-09-17T09:46:41.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:41 smithi078 ceph-mon[63787]: pgmap v198: 97 pgs: 97 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.8 MiB/s wr, 381 op/s; 225 KiB/s, 37 objects/s recovering 2024-09-17T09:46:41.848 INFO:teuthology.orchestra.run.smithi078.stdout: 5 244402 10.09 MB/sec execute 456 sec latency 42.380 ms 2024-09-17T09:46:41.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:41 smithi136 ceph-mon[49560]: pgmap v198: 97 pgs: 97 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.8 MiB/s wr, 381 op/s; 225 KiB/s, 37 objects/s recovering 2024-09-17T09:46:42.848 INFO:teuthology.orchestra.run.smithi078.stdout: 5 244757 10.09 MB/sec execute 457 sec latency 62.079 ms 2024-09-17T09:46:43.656 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:43 smithi078 ceph-mon[63787]: pgmap v199: 97 pgs: 97 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 3.0 MiB/s wr, 314 op/s; 8.5 KiB/s, 7 objects/s recovering 2024-09-17T09:46:43.848 INFO:teuthology.orchestra.run.smithi078.stdout: 5 245071 10.09 MB/sec execute 458 sec latency 51.023 ms 2024-09-17T09:46:43.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:43 smithi136 ceph-mon[49560]: pgmap v199: 97 pgs: 97 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 3.0 MiB/s wr, 314 op/s; 8.5 KiB/s, 7 objects/s recovering 2024-09-17T09:46:44.848 INFO:teuthology.orchestra.run.smithi078.stdout: 5 245381 10.08 MB/sec execute 459 sec latency 59.246 ms 2024-09-17T09:46:45.707 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:45 smithi078 ceph-mon[63787]: pgmap v200: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.5 MiB/s wr, 419 op/s; 6.8 KiB/s, 6 objects/s recovering 2024-09-17T09:46:45.848 INFO:teuthology.orchestra.run.smithi078.stdout: 5 245714 10.07 MB/sec execute 460 sec latency 51.693 ms 2024-09-17T09:46:45.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:45 smithi136 ceph-mon[49560]: pgmap v200: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.5 MiB/s wr, 419 op/s; 6.8 KiB/s, 6 objects/s recovering 2024-09-17T09:46:46.725 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:46 smithi078 ceph-mon[63787]: pgmap v201: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.1 MiB/s wr, 381 op/s; 6.2 KiB/s, 5 objects/s recovering 2024-09-17T09:46:46.848 INFO:teuthology.orchestra.run.smithi078.stdout: 5 246009 10.06 MB/sec execute 461 sec latency 47.199 ms 2024-09-17T09:46:46.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:46 smithi136 ceph-mon[49560]: pgmap v201: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.1 MiB/s wr, 381 op/s; 6.2 KiB/s, 5 objects/s recovering 2024-09-17T09:46:47.748 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:47 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-17T09:46:47.848 INFO:teuthology.orchestra.run.smithi078.stdout: 5 246283 10.06 MB/sec execute 462 sec latency 52.020 ms 2024-09-17T09:46:47.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:47 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-17T09:46:48.815 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:48 smithi078 ceph-mon[63787]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T09:46:48.816 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:48 smithi078 ceph-mon[63787]: Upgrade: osd.4 is safe to restart 2024-09-17T09:46:48.816 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:48 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:48.816 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:48 smithi078 ceph-mon[63787]: pgmap v202: 97 pgs: 97 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.9 MiB/s wr, 511 op/s; 5.7 KiB/s, 5 objects/s recovering 2024-09-17T09:46:48.848 INFO:teuthology.orchestra.run.smithi078.stdout: 5 246629 10.07 MB/sec execute 463 sec latency 58.357 ms 2024-09-17T09:46:49.154 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:48 smithi136 ceph-mon[49560]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T09:46:49.154 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:48 smithi136 ceph-mon[49560]: Upgrade: osd.4 is safe to restart 2024-09-17T09:46:49.155 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:48 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:49.155 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:48 smithi136 ceph-mon[49560]: pgmap v202: 97 pgs: 97 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.9 MiB/s wr, 511 op/s; 5.7 KiB/s, 5 objects/s recovering 2024-09-17T09:46:49.848 INFO:teuthology.orchestra.run.smithi078.stdout: 5 246872 10.06 MB/sec execute 464 sec latency 82.959 ms 2024-09-17T09:46:50.432 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:50 smithi078 ceph-mon[63787]: Upgrade: Updating osd.4 2024-09-17T09:46:50.432 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:50 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:50.432 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:50 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-17T09:46:50.432 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:50 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:46:50.432 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:50 smithi078 ceph-mon[63787]: Deploying daemon osd.4 on smithi136 2024-09-17T09:46:50.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:50 smithi136 ceph-mon[49560]: Upgrade: Updating osd.4 2024-09-17T09:46:50.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:50 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:50.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:50 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-17T09:46:50.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:50 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:46:50.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:50 smithi136 ceph-mon[49560]: Deploying daemon osd.4 on smithi136 2024-09-17T09:46:50.849 INFO:teuthology.orchestra.run.smithi078.stdout: 5 247178 10.05 MB/sec execute 465 sec latency 47.535 ms 2024-09-17T09:46:51.224 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:51 smithi078 ceph-mon[63787]: pgmap v203: 97 pgs: 97 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.0 MiB/s wr, 464 op/s; 0 B/s, 1 objects/s recovering 2024-09-17T09:46:51.312 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:51 smithi136 ceph-mon[49560]: pgmap v203: 97 pgs: 97 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.0 MiB/s wr, 464 op/s; 0 B/s, 1 objects/s recovering 2024-09-17T09:46:51.849 INFO:teuthology.orchestra.run.smithi078.stdout: 5 247387 10.04 MB/sec execute 466 sec latency 71.499 ms 2024-09-17T09:46:52.726 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:52 smithi136 systemd[1]: Stopping Ceph osd.4 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:46:52.849 INFO:teuthology.orchestra.run.smithi078.stdout: 5 247576 10.03 MB/sec execute 467 sec latency 68.392 ms 2024-09-17T09:46:53.226 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:52 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4[30198]: 2024-09-17T09:46:52.831+0000 7f9f6055e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:46:53.226 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:52 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4[30198]: 2024-09-17T09:46:52.831+0000 7f9f6055e700 -1 osd.4 124 *** Got signal Terminated *** 2024-09-17T09:46:53.226 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:52 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4[30198]: 2024-09-17T09:46:52.831+0000 7f9f6055e700 -1 osd.4 124 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:46:53.545 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:53 smithi078 ceph-mon[63787]: pgmap v204: 97 pgs: 97 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.3 MiB/s wr, 436 op/s 2024-09-17T09:46:53.546 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:53 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:53.546 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:53 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:46:53.546 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:53 smithi078 ceph-mon[63787]: osd.4 marked itself down and dead 2024-09-17T09:46:53.849 INFO:teuthology.orchestra.run.smithi078.stdout: 5 247809 10.03 MB/sec execute 468 sec latency 137.688 ms 2024-09-17T09:46:53.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:53 smithi136 ceph-mon[49560]: pgmap v204: 97 pgs: 97 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.3 MiB/s wr, 436 op/s 2024-09-17T09:46:53.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:53 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:46:53.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:53 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:46:53.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:53 smithi136 ceph-mon[49560]: osd.4 marked itself down and dead 2024-09-17T09:46:54.649 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:54 smithi136 ceph-mon[49560]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:46:54.649 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:54 smithi136 ceph-mon[49560]: osdmap e125: 8 total, 7 up, 8 in 2024-09-17T09:46:54.650 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:54 smithi136 podman[52968]: 2024-09-17 09:46:54.351798825 +0000 UTC m=+1.645548281 container died a0c5b4de2414e17897564b3eeec0c1f5ccc014571542d65ae49e486da232669a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, ceph=True, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , vcs-type=git, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, architecture=x86_64, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8) 2024-09-17T09:46:54.787 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:54 smithi078 ceph-mon[63787]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:46:54.787 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:54 smithi078 ceph-mon[63787]: osdmap e125: 8 total, 7 up, 8 in 2024-09-17T09:46:54.849 INFO:teuthology.orchestra.run.smithi078.stdout: 5 247812 10.01 MB/sec execute 469 sec latency 1078.376 ms 2024-09-17T09:46:54.975 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:54 smithi136 podman[52968]: 2024-09-17 09:46:54.756726095 +0000 UTC m=+2.050475559 container cleanup a0c5b4de2414e17897564b3eeec0c1f5ccc014571542d65ae49e486da232669a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, vendor=Red Hat, Inc., GIT_CLEAN=True, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, RELEASE=HEAD, maintainer=Guillaume Abrioux , distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, io.k8s.display-name=CentOS Stream 8) 2024-09-17T09:46:54.976 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:54 smithi136 bash[52968]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4 2024-09-17T09:46:55.476 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:55 smithi136 podman[52986]: 2024-09-17 09:46:55.050023201 +0000 UTC m=+0.695103620 container remove a0c5b4de2414e17897564b3eeec0c1f5ccc014571542d65ae49e486da232669a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, vcs-type=git, ceph=True, name=centos-stream, release=754, GIT_CLEAN=True, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-17T09:46:55.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:55 smithi078 ceph-mon[63787]: pgmap v206: 97 pgs: 8 peering, 12 stale+active+clean, 77 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 462 op/s 2024-09-17T09:46:55.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:55 smithi078 ceph-mon[63787]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-17T09:46:55.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:55 smithi078 ceph-mon[63787]: osdmap e126: 8 total, 7 up, 8 in 2024-09-17T09:46:55.775 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:55 smithi136 ceph-mon[49560]: pgmap v206: 97 pgs: 8 peering, 12 stale+active+clean, 77 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 462 op/s 2024-09-17T09:46:55.775 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:55 smithi136 ceph-mon[49560]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-17T09:46:55.775 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:55 smithi136 ceph-mon[49560]: osdmap e126: 8 total, 7 up, 8 in 2024-09-17T09:46:55.775 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:55 smithi136 podman[53072]: 2024-09-17 09:46:55.519200897 +0000 UTC m=+0.024057823 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:46:55.775 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:55 smithi136 podman[53072]: 2024-09-17 09:46:55.650950938 +0000 UTC m=+0.155807856 container create b109dd7f059acd5464e1ad098768a9dd11ee8e2da1e6e0d45021b5e91555cade (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_CLEAN=True) 2024-09-17T09:46:55.849 INFO:teuthology.orchestra.run.smithi078.stdout: 5 248141 10.00 MB/sec execute 470 sec latency 64.383 ms 2024-09-17T09:46:56.147 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:55 smithi136 podman[53072]: 2024-09-17 09:46:55.911989187 +0000 UTC m=+0.416846102 container init b109dd7f059acd5464e1ad098768a9dd11ee8e2da1e6e0d45021b5e91555cade (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-17T09:46:56.147 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:55 smithi136 podman[53072]: 2024-09-17 09:46:55.916456278 +0000 UTC m=+0.421313193 container start b109dd7f059acd5464e1ad098768a9dd11ee8e2da1e6e0d45021b5e91555cade (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2) 2024-09-17T09:46:56.147 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:56 smithi136 podman[53072]: 2024-09-17 09:46:56.003767089 +0000 UTC m=+0.508624004 container attach b109dd7f059acd5464e1ad098768a9dd11ee8e2da1e6e0d45021b5e91555cade (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-17T09:46:56.475 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:56 smithi136 podman[53072]: 2024-09-17 09:46:56.146160927 +0000 UTC m=+0.651017854 container died b109dd7f059acd5464e1ad098768a9dd11ee8e2da1e6e0d45021b5e91555cade (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4-deactivate, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-17T09:46:56.849 INFO:teuthology.orchestra.run.smithi078.stdout: 5 248458 10.00 MB/sec execute 471 sec latency 65.716 ms 2024-09-17T09:46:57.226 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:56 smithi136 podman[53093]: 2024-09-17 09:46:56.967829755 +0000 UTC m=+0.810610914 container remove b109dd7f059acd5464e1ad098768a9dd11ee8e2da1e6e0d45021b5e91555cade (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-17T09:46:57.226 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:56 smithi136 systemd[1]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.4.service: Deactivated successfully. 2024-09-17T09:46:57.226 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:56 smithi136 systemd[1]: Stopped Ceph osd.4 for 35c0cd22-74d7-11ef-bceb-c7b262605968. 2024-09-17T09:46:57.226 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:56 smithi136 systemd[1]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.4.service: Consumed 4min 40.149s CPU time. 2024-09-17T09:46:57.508 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:57 smithi136 systemd[1]: Starting Ceph osd.4 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:46:57.508 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:57 smithi136 podman[53193]: 2024-09-17 09:46:57.36976188 +0000 UTC m=+0.018932988 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:46:57.676 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:57 smithi078 ceph-mon[63787]: pgmap v208: 97 pgs: 8 peering, 12 stale+active+clean, 77 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.6 MiB/s wr, 335 op/s 2024-09-17T09:46:57.765 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:57 smithi136 ceph-mon[49560]: pgmap v208: 97 pgs: 8 peering, 12 stale+active+clean, 77 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.6 MiB/s wr, 335 op/s 2024-09-17T09:46:57.765 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:57 smithi136 podman[53193]: 2024-09-17 09:46:57.510962875 +0000 UTC m=+0.160133958 container create e90aeb304ec5fc2e011b1beff2a8f50a2389e6416d26679860bc9afe0e124634 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T09:46:57.849 INFO:teuthology.orchestra.run.smithi078.stdout: 5 248738 10.00 MB/sec execute 472 sec latency 49.577 ms 2024-09-17T09:46:58.226 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:57 smithi136 podman[53193]: 2024-09-17 09:46:57.764209887 +0000 UTC m=+0.413380984 container init e90aeb304ec5fc2e011b1beff2a8f50a2389e6416d26679860bc9afe0e124634 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4-activate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72) 2024-09-17T09:46:58.226 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:57 smithi136 podman[53193]: 2024-09-17 09:46:57.769050175 +0000 UTC m=+0.418221265 container start e90aeb304ec5fc2e011b1beff2a8f50a2389e6416d26679860bc9afe0e124634 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-17T09:46:58.226 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:57 smithi136 podman[53193]: 2024-09-17 09:46:57.880777287 +0000 UTC m=+0.529948378 container attach e90aeb304ec5fc2e011b1beff2a8f50a2389e6416d26679860bc9afe0e124634 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4-activate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-17T09:46:58.226 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:57 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4-activate[53207]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:46:58.226 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:57 smithi136 bash[53193]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:46:58.226 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:57 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4-activate[53207]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:46:58.226 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:57 smithi136 bash[53193]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:46:58.714 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:58 smithi078 ceph-mon[63787]: pgmap v209: 97 pgs: 8 active+undersized, 8 peering, 5 stale+active+clean, 6 active+undersized+degraded, 70 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.2 MiB/s wr, 288 op/s; 66/1578 objects degraded (4.183%) 2024-09-17T09:46:58.715 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:46:58 smithi078 ceph-mon[63787]: Health check failed: Degraded data redundancy: 66/1578 objects degraded (4.183%), 6 pgs degraded (PG_DEGRADED) 2024-09-17T09:46:58.849 INFO:teuthology.orchestra.run.smithi078.stdout: 5 248993 9.98 MB/sec execute 473 sec latency 82.013 ms 2024-09-17T09:46:58.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:58 smithi136 ceph-mon[49560]: pgmap v209: 97 pgs: 8 active+undersized, 8 peering, 5 stale+active+clean, 6 active+undersized+degraded, 70 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.2 MiB/s wr, 288 op/s; 66/1578 objects degraded (4.183%) 2024-09-17T09:46:58.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:46:58 smithi136 ceph-mon[49560]: Health check failed: Degraded data redundancy: 66/1578 objects degraded (4.183%), 6 pgs degraded (PG_DEGRADED) 2024-09-17T09:46:59.609 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:59 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4-activate[53207]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-17T09:46:59.609 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:59 smithi136 bash[53193]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-17T09:46:59.609 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:59 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4-activate[53207]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-17T09:46:59.609 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:59 smithi136 bash[53193]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-17T09:46:59.609 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:59 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4-activate[53207]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-17T09:46:59.609 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:59 smithi136 bash[53193]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-17T09:46:59.849 INFO:teuthology.orchestra.run.smithi078.stdout: 5 249340 9.98 MB/sec execute 474 sec latency 55.367 ms 2024-09-17T09:46:59.976 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:59 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4-activate[53207]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-17T09:46:59.976 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:59 smithi136 bash[53193]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-17T09:46:59.976 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:59 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4-activate[53207]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-17T09:46:59.976 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:59 smithi136 bash[53193]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-17T09:46:59.976 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:59 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4-activate[53207]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-17T09:46:59.976 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:59 smithi136 bash[53193]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-17T09:46:59.976 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:59 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4-activate[53207]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-17T09:46:59.976 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:59 smithi136 bash[53193]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-17T09:46:59.976 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:46:59 smithi136 podman[53193]: 2024-09-17 09:46:59.643256243 +0000 UTC m=+2.292427326 container died e90aeb304ec5fc2e011b1beff2a8f50a2389e6416d26679860bc9afe0e124634 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4-activate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72) 2024-09-17T09:47:00.680 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:00 smithi136 ceph-mon[49560]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-17T09:47:00.681 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:47:00 smithi136 podman[53339]: 2024-09-17 09:47:00.428625736 +0000 UTC m=+0.778205012 container remove e90aeb304ec5fc2e011b1beff2a8f50a2389e6416d26679860bc9afe0e124634 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True) 2024-09-17T09:47:00.681 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:47:00 smithi136 podman[53397]: 2024-09-17 09:47:00.556197809 +0000 UTC m=+0.020614344 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:47:00.778 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:00 smithi078 ceph-mon[63787]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-17T09:47:00.849 INFO:teuthology.orchestra.run.smithi078.stdout: 5 249571 9.97 MB/sec execute 475 sec latency 50.038 ms 2024-09-17T09:47:00.934 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:47:00 smithi136 podman[53397]: 2024-09-17 09:47:00.679930677 +0000 UTC m=+0.144347192 container create e2a18ad9466438e628af3d470891ea0615e10c803666172b228adffff0ecee82 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T09:47:01.226 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:47:00 smithi136 podman[53397]: 2024-09-17 09:47:00.932766508 +0000 UTC m=+0.397183023 container init e2a18ad9466438e628af3d470891ea0615e10c803666172b228adffff0ecee82 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-17T09:47:01.226 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:47:00 smithi136 podman[53397]: 2024-09-17 09:47:00.937325625 +0000 UTC m=+0.401742141 container start e2a18ad9466438e628af3d470891ea0615e10c803666172b228adffff0ecee82 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T09:47:01.226 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:47:00 smithi136 ceph-osd[53416]: -- 172.21.15.136:0/832139772 <== mon.1 v2:172.21.15.136:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ba24ade1e0 con 0x55ba24a68000 2024-09-17T09:47:01.226 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:47:01 smithi136 bash[53397]: e2a18ad9466438e628af3d470891ea0615e10c803666172b228adffff0ecee82 2024-09-17T09:47:01.226 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:47:01 smithi136 systemd[1]: Started Ceph osd.4 for 35c0cd22-74d7-11ef-bceb-c7b262605968. 2024-09-17T09:47:01.801 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:01 smithi078 ceph-mon[63787]: pgmap v210: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.2 MiB/s wr, 439 op/s; 262/1581 objects degraded (16.572%) 2024-09-17T09:47:01.802 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:01 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:01.802 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:01 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:01.802 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:01 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:47:01.849 INFO:teuthology.orchestra.run.smithi078.stdout: 5 249837 9.96 MB/sec execute 476 sec latency 63.374 ms 2024-09-17T09:47:01.860 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:01 smithi136 ceph-mon[49560]: pgmap v210: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 336 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.2 MiB/s wr, 439 op/s; 262/1581 objects degraded (16.572%) 2024-09-17T09:47:01.860 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:01 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:01.860 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:01 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:01.861 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:01 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:47:02.849 INFO:teuthology.orchestra.run.smithi078.stdout: 5 250154 9.97 MB/sec execute 477 sec latency 92.262 ms 2024-09-17T09:47:03.075 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:47:02 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4[53412]: 2024-09-17T09:47:02.836+0000 7f8328b64640 -1 Falling back to public interface 2024-09-17T09:47:03.588 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:03 smithi078 ceph-mon[63787]: pgmap v211: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 2.5 MiB/s wr, 255 op/s; 255/1578 objects degraded (16.160%) 2024-09-17T09:47:03.849 INFO:teuthology.orchestra.run.smithi078.stdout: 5 250447 9.96 MB/sec execute 478 sec latency 73.343 ms 2024-09-17T09:47:03.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:03 smithi136 ceph-mon[49560]: pgmap v211: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 2.5 MiB/s wr, 255 op/s; 255/1578 objects degraded (16.160%) 2024-09-17T09:47:04.849 INFO:teuthology.orchestra.run.smithi078.stdout: 5 250737 9.96 MB/sec execute 479 sec latency 73.073 ms 2024-09-17T09:47:04.999 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-17T09:47:05.678 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:05 smithi078 ceph-mon[63787]: pgmap v212: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.6 MiB/s wr, 354 op/s; 255/1578 objects degraded (16.160%) 2024-09-17T09:47:05.679 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:05 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:05.679 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:05 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:05.849 INFO:teuthology.orchestra.run.smithi078.stdout: 5 250937 9.95 MB/sec execute 480 sec latency 68.643 ms 2024-09-17T09:47:05.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:05 smithi136 ceph-mon[49560]: pgmap v212: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.6 MiB/s wr, 354 op/s; 255/1578 objects degraded (16.160%) 2024-09-17T09:47:05.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:05 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:05.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:05 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:06.077 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:47:06.077 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (7m) 39s ago 16m 17.8M - 0.27.0 11f11916f8cd 97db2373ce92 2024-09-17T09:47:06.077 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (4m) 39s ago 16m 6899k - 19.3.0-4963-ge646f729 f271290a0950 da528c5ca445 2024-09-17T09:47:06.077 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi136 smithi136 running (3m) 1s ago 15m 6899k - 19.3.0-4963-ge646f729 f271290a0950 909f405348cc 2024-09-17T09:47:06.077 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (7m) 39s ago 16m 53.0M - dad864ee21e9 db74b0f8e0a7 2024-09-17T09:47:06.077 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.jdrsup smithi078 running (10m) 39s ago 10m 38.4M - 17.2.0 e1d6a67b021e 8f834e15ba88 2024-09-17T09:47:06.078 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi136.smslff smithi136 running (10m) 1s ago 10m 16.5M - 17.2.0 e1d6a67b021e 0c20e2b328d3 2024-09-17T09:47:06.078 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rsqofi smithi078 *:8443,9283,8765 running (6m) 39s ago 17m 519M - 19.3.0-4963-ge646f729 f271290a0950 7dda3108c9a0 2024-09-17T09:47:06.078 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi136.hicmlm smithi136 *:8443,9283,8765 running (5m) 1s ago 15m 456M - 19.3.0-4963-ge646f729 f271290a0950 fc518b58e07a 2024-09-17T09:47:06.078 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (5m) 39s ago 18m 53.2M 2048M 19.3.0-4963-ge646f729 f271290a0950 3ead101d3db1 2024-09-17T09:47:06.078 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi136 smithi136 running (4m) 1s ago 15m 42.2M 2048M 19.3.0-4963-ge646f729 f271290a0950 4fce88842b79 2024-09-17T09:47:06.078 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.tcwbvi smithi078 *:2049 running (10m) 39s ago 10m 284M - 4.0 e1d6a67b021e 30c87cb46555 2024-09-17T09:47:06.078 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi136.regjvc smithi136 *:2049 running (10m) 1s ago 10m 90.2M - 4.0 e1d6a67b021e b88e4dc24376 2024-09-17T09:47:06.078 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (7m) 39s ago 16m 11.0M - 1.7.0 72c9c2088986 52dad3adeeaa 2024-09-17T09:47:06.078 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi136 smithi136 *:9100 running (6m) 1s ago 15m 11.8M - 1.7.0 72c9c2088986 f2bc5856e35a 2024-09-17T09:47:06.078 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (3m) 39s ago 15m 351M 1936M 19.3.0-4963-ge646f729 f271290a0950 1ea81cc809b8 2024-09-17T09:47:06.078 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (2m) 39s ago 14m 290M 1936M 19.3.0-4963-ge646f729 f271290a0950 eda32560334b 2024-09-17T09:47:06.079 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (100s) 39s ago 14m 213M 1936M 19.3.0-4963-ge646f729 f271290a0950 83a32ad7c358 2024-09-17T09:47:06.079 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (51s) 39s ago 13m 14.3M 1936M 19.3.0-4963-ge646f729 f271290a0950 e2cd18998d2b 2024-09-17T09:47:06.079 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi136 running (5s) 1s ago 13m 11.6M 2704M 19.3.0-4963-ge646f729 f271290a0950 e2a18ad94664 2024-09-17T09:47:06.079 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi136 running (12m) 1s ago 12m 592M 2704M 17.2.0 e1d6a67b021e 30a3049955f9 2024-09-17T09:47:06.079 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi136 running (12m) 1s ago 12m 530M 2704M 17.2.0 e1d6a67b021e 1698e1567c33 2024-09-17T09:47:06.079 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi136 running (12m) 1s ago 12m 796M 2704M 17.2.0 e1d6a67b021e 5d3b08cecd78 2024-09-17T09:47:06.079 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (5m) 39s ago 15m 53.8M - 2.51.0 1d3b7f56885b 6e5e8c721eb0 2024-09-17T09:47:06.714 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:47:06.714 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-17T09:47:06.714 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:47:06.715 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:47:06.715 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-17T09:47:06.715 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:47:06.715 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:47:06.715 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-17T09:47:06.715 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-17T09:47:06.715 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 4 2024-09-17T09:47:06.715 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:47:06.715 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-17T09:47:06.715 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:47:06.715 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:47:06.715 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-17T09:47:06.715 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-17T09:47:06.715 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 8 2024-09-17T09:47:06.715 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-17T09:47:06.716 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:47:06.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:06 smithi078 ceph-mon[63787]: from='client.34356 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:47:06.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:06 smithi078 ceph-mon[63787]: from='client.34360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:47:06.730 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:06 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:06.730 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:06 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:06.730 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:06 smithi078 ceph-mon[63787]: from='client.34364 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:47:06.730 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:06 smithi078 ceph-mon[63787]: pgmap v213: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.1 MiB/s wr, 303 op/s; 255/1578 objects degraded (16.160%) 2024-09-17T09:47:06.849 INFO:teuthology.orchestra.run.smithi078.stdout: 5 251144 9.94 MB/sec execute 481 sec latency 64.303 ms 2024-09-17T09:47:06.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:06 smithi136 ceph-mon[49560]: from='client.34356 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:47:06.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:06 smithi136 ceph-mon[49560]: from='client.34360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:47:06.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:06 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:06.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:06 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:06.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:06 smithi136 ceph-mon[49560]: from='client.34364 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:47:06.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:06 smithi136 ceph-mon[49560]: pgmap v213: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 337 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.1 MiB/s wr, 303 op/s; 255/1578 objects degraded (16.160%) 2024-09-17T09:47:07.259 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:47:07.259 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407", 2024-09-17T09:47:07.259 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-17T09:47:07.259 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T09:47:07.260 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-17T09:47:07.260 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-09-17T09:47:07.260 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-17T09:47:07.260 INFO:teuthology.orchestra.run.smithi078.stdout: "mon" 2024-09-17T09:47:07.260 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-17T09:47:07.260 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "11/23 daemons upgraded", 2024-09-17T09:47:07.260 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T09:47:07.260 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-17T09:47:07.260 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:47:07.765 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:07 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/3644994412' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:47:07.765 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:07 smithi078 ceph-mon[63787]: from='client.44287 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:47:07.765 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:07 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 255/1578 objects degraded (16.160%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T09:47:07.849 INFO:teuthology.orchestra.run.smithi078.stdout: 5 251344 9.93 MB/sec execute 482 sec latency 73.841 ms 2024-09-17T09:47:07.929 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 255/1578 objects degraded (16.160%), 29 pgs degraded 2024-09-17T09:47:07.930 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-17T09:47:07.930 INFO:teuthology.orchestra.run.smithi078.stdout: osd.4 (root=default,host=smithi136) is down 2024-09-17T09:47:07.930 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 255/1578 objects degraded (16.160%), 29 pgs degraded 2024-09-17T09:47:07.930 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.3 is active+undersized+degraded, acting [5,2] 2024-09-17T09:47:07.930 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.5 is active+undersized+degraded, acting [7,0] 2024-09-17T09:47:07.930 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.6 is active+undersized+degraded, acting [1,6] 2024-09-17T09:47:07.930 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.b is active+undersized+degraded, acting [7,5] 2024-09-17T09:47:07.930 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.d is active+undersized+degraded, acting [1,3] 2024-09-17T09:47:07.930 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.f is active+undersized+degraded, acting [0,7] 2024-09-17T09:47:07.931 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.11 is active+undersized+degraded, acting [6,1] 2024-09-17T09:47:07.931 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.13 is active+undersized+degraded, acting [0,3] 2024-09-17T09:47:07.931 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.18 is active+undersized+degraded, acting [5,7] 2024-09-17T09:47:07.931 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1a is active+undersized+degraded, acting [6,7] 2024-09-17T09:47:07.931 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1f is active+undersized+degraded, acting [0,7] 2024-09-17T09:47:07.931 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1 is active+undersized+degraded, acting [0,3] 2024-09-17T09:47:07.931 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.3 is active+undersized+degraded, acting [0,6] 2024-09-17T09:47:07.931 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.6 is active+undersized+degraded, acting [0,1] 2024-09-17T09:47:07.931 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.9 is active+undersized+degraded, acting [2,7] 2024-09-17T09:47:07.932 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.a is active+undersized+degraded, acting [6,1] 2024-09-17T09:47:07.932 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.b is active+undersized+degraded, acting [3,0] 2024-09-17T09:47:07.932 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.e is active+undersized+degraded, acting [7,1] 2024-09-17T09:47:07.932 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.f is active+undersized+degraded, acting [7,0] 2024-09-17T09:47:07.932 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.11 is active+undersized+degraded, acting [7,6] 2024-09-17T09:47:07.932 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.13 is active+undersized+degraded, acting [7,2] 2024-09-17T09:47:07.932 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.14 is active+undersized+degraded, acting [7,6] 2024-09-17T09:47:07.932 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.15 is active+undersized+degraded, acting [7,3] 2024-09-17T09:47:07.932 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.19 is active+undersized+degraded, acting [1,3] 2024-09-17T09:47:07.932 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1a is active+undersized+degraded, acting [1,2] 2024-09-17T09:47:07.932 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1b is active+undersized+degraded, acting [0,7] 2024-09-17T09:47:07.932 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1c is active+undersized+degraded, acting [5,1] 2024-09-17T09:47:07.932 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1d is active+undersized+degraded, acting [5,6] 2024-09-17T09:47:07.933 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.5 is active+undersized+degraded, acting [6,7] 2024-09-17T09:47:07.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:07 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/3644994412' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:47:07.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:07 smithi136 ceph-mon[49560]: from='client.44287 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:47:07.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:07 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 255/1578 objects degraded (16.160%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T09:47:08.783 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:08 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:08.783 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:08 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:47:08.783 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:08 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/536259736' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:47:08.783 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:08 smithi078 ceph-mon[63787]: pgmap v214: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.9 MiB/s wr, 416 op/s; 258/1599 objects degraded (16.135%) 2024-09-17T09:47:08.849 INFO:teuthology.orchestra.run.smithi078.stdout: 5 251593 9.92 MB/sec execute 483 sec latency 68.585 ms 2024-09-17T09:47:08.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:08 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:08.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:08 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:47:08.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:08 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/536259736' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:47:08.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:08 smithi136 ceph-mon[49560]: pgmap v214: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.9 MiB/s wr, 416 op/s; 258/1599 objects degraded (16.135%) 2024-09-17T09:47:09.849 INFO:teuthology.orchestra.run.smithi078.stdout: 5 251822 9.92 MB/sec execute 484 sec latency 67.051 ms 2024-09-17T09:47:09.975 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:47:09 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4[53412]: 2024-09-17T09:47:09.535+0000 7f8328b64640 -1 osd.4 0 read_superblock omap replica is missing. 2024-09-17T09:47:10.325 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:10 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:10.326 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:10 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:10.326 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:10 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:47:10.326 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:10 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:47:10.326 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:10 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:10.326 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:10 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:10.326 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:10 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:47:10.326 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:10 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:47:10.326 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:10 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:47:10.326 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:10 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:47:10.326 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:10 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-17T09:47:10.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:10 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:10.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:10 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:10.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:10 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:47:10.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:10 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:47:10.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:10 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:10.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:10 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:10.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:10 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:47:10.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:10 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:47:10.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:10 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:47:10.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:10 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:47:10.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:10 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-17T09:47:10.849 INFO:teuthology.orchestra.run.smithi078.stdout: 5 252105 9.91 MB/sec execute 485 sec latency 81.655 ms 2024-09-17T09:47:11.212 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:47:10 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4[53412]: 2024-09-17T09:47:10.796+0000 7f8328b64640 -1 osd.4 124 log_to_monitors true 2024-09-17T09:47:11.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:11 smithi078 ceph-mon[63787]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T09:47:11.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:11 smithi078 ceph-mon[63787]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-09-17T09:47:11.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:11 smithi078 ceph-mon[63787]: pgmap v215: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.1 MiB/s wr, 426 op/s; 258/1593 objects degraded (16.196%); 172 B/s, 0 objects/s recovering 2024-09-17T09:47:11.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:11 smithi078 ceph-mon[63787]: from='osd.4 [v2:172.21.15.136:6800/1573919959,v1:172.21.15.136:6801/1573919959]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-17T09:47:11.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:11 smithi078 ceph-mon[63787]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-17T09:47:11.478 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:47:11 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4[53412]: 2024-09-17T09:47:11.230+0000 7f832090e640 -1 osd.4 124 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T09:47:11.479 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:11 smithi136 ceph-mon[49560]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T09:47:11.479 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:11 smithi136 ceph-mon[49560]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-09-17T09:47:11.479 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:11 smithi136 ceph-mon[49560]: pgmap v215: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 340 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.1 MiB/s wr, 426 op/s; 258/1593 objects degraded (16.196%); 172 B/s, 0 objects/s recovering 2024-09-17T09:47:11.479 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:11 smithi136 ceph-mon[49560]: from='osd.4 [v2:172.21.15.136:6800/1573919959,v1:172.21.15.136:6801/1573919959]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-17T09:47:11.479 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:11 smithi136 ceph-mon[49560]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-17T09:47:11.849 INFO:teuthology.orchestra.run.smithi078.stdout: 5 252541 9.91 MB/sec execute 486 sec latency 43.475 ms 2024-09-17T09:47:12.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:12 smithi078 ceph-mon[63787]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-17T09:47:12.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:12 smithi078 ceph-mon[63787]: osdmap e127: 8 total, 7 up, 8 in 2024-09-17T09:47:12.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:12 smithi078 ceph-mon[63787]: from='osd.4 [v2:172.21.15.136:6800/1573919959,v1:172.21.15.136:6801/1573919959]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi136", "root=default"]} : dispatch 2024-09-17T09:47:12.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:12 smithi078 ceph-mon[63787]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi136", "root=default"]} : dispatch 2024-09-17T09:47:12.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:12 smithi136 ceph-mon[49560]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-17T09:47:12.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:12 smithi136 ceph-mon[49560]: osdmap e127: 8 total, 7 up, 8 in 2024-09-17T09:47:12.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:12 smithi136 ceph-mon[49560]: from='osd.4 [v2:172.21.15.136:6800/1573919959,v1:172.21.15.136:6801/1573919959]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi136", "root=default"]} : dispatch 2024-09-17T09:47:12.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:12 smithi136 ceph-mon[49560]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi136", "root=default"]} : dispatch 2024-09-17T09:47:12.849 INFO:teuthology.orchestra.run.smithi078.stdout: 5 252765 9.90 MB/sec execute 487 sec latency 643.235 ms 2024-09-17T09:47:13.513 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:13 smithi078 ceph-mon[63787]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:47:13.514 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:13 smithi078 ceph-mon[63787]: osd.4 [v2:172.21.15.136:6800/1573919959,v1:172.21.15.136:6801/1573919959] boot 2024-09-17T09:47:13.514 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:13 smithi078 ceph-mon[63787]: osdmap e128: 8 total, 8 up, 8 in 2024-09-17T09:47:13.514 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:13 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-17T09:47:13.514 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:13 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 258/1593 objects degraded (16.196%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T09:47:13.514 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:13 smithi078 ceph-mon[63787]: pgmap v219: 97 pgs: 5 peering, 11 active+undersized, 26 active+undersized+degraded, 55 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.0 MiB/s wr, 434 op/s; 235/1605 objects degraded (14.642%) 2024-09-17T09:47:13.514 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:13 smithi078 ceph-mon[63787]: osdmap e129: 8 total, 8 up, 8 in 2024-09-17T09:47:13.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:13 smithi136 ceph-mon[49560]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:47:13.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:13 smithi136 ceph-mon[49560]: osd.4 [v2:172.21.15.136:6800/1573919959,v1:172.21.15.136:6801/1573919959] boot 2024-09-17T09:47:13.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:13 smithi136 ceph-mon[49560]: osdmap e128: 8 total, 8 up, 8 in 2024-09-17T09:47:13.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:13 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-17T09:47:13.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:13 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 258/1593 objects degraded (16.196%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T09:47:13.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:13 smithi136 ceph-mon[49560]: pgmap v219: 97 pgs: 5 peering, 11 active+undersized, 26 active+undersized+degraded, 55 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.0 MiB/s wr, 434 op/s; 235/1605 objects degraded (14.642%) 2024-09-17T09:47:13.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:13 smithi136 ceph-mon[49560]: osdmap e129: 8 total, 8 up, 8 in 2024-09-17T09:47:13.849 INFO:teuthology.orchestra.run.smithi078.stdout: 5 252929 9.89 MB/sec execute 488 sec latency 1301.761 ms 2024-09-17T09:47:14.549 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:14 smithi078 ceph-mon[63787]: osdmap e130: 8 total, 8 up, 8 in 2024-09-17T09:47:14.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:14 smithi136 ceph-mon[49560]: osdmap e130: 8 total, 8 up, 8 in 2024-09-17T09:47:14.850 INFO:teuthology.orchestra.run.smithi078.stdout: 5 253169 9.88 MB/sec execute 489 sec latency 351.515 ms 2024-09-17T09:47:15.719 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:15 smithi078 ceph-mon[63787]: pgmap v221: 97 pgs: 1 active+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 activating+undersized+remapped, 1 remapped+peering, 5 peering, 9 active+undersized, 5 active+undersized+degraded, 72 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.7 MiB/s wr, 564 op/s; 94/1602 objects degraded (5.868%); 971 KiB/s, 37 objects/s recovering 2024-09-17T09:47:15.719 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:15 smithi078 ceph-mon[63787]: osdmap e131: 8 total, 8 up, 8 in 2024-09-17T09:47:15.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:15 smithi136 ceph-mon[49560]: pgmap v221: 97 pgs: 1 active+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 activating+undersized+remapped, 1 remapped+peering, 5 peering, 9 active+undersized, 5 active+undersized+degraded, 72 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.7 MiB/s wr, 564 op/s; 94/1602 objects degraded (5.868%); 971 KiB/s, 37 objects/s recovering 2024-09-17T09:47:15.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:15 smithi136 ceph-mon[49560]: osdmap e131: 8 total, 8 up, 8 in 2024-09-17T09:47:15.849 INFO:teuthology.orchestra.run.smithi078.stdout: 5 253313 9.87 MB/sec execute 490 sec latency 981.223 ms 2024-09-17T09:47:16.486 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:16 smithi078 ceph-mon[63787]: osdmap e132: 8 total, 8 up, 8 in 2024-09-17T09:47:16.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:16 smithi136 ceph-mon[49560]: osdmap e132: 8 total, 8 up, 8 in 2024-09-17T09:47:16.849 INFO:teuthology.orchestra.run.smithi078.stdout: 5 253675 9.88 MB/sec execute 491 sec latency 37.467 ms 2024-09-17T09:47:17.517 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:17 smithi078 ceph-mon[63787]: pgmap v224: 97 pgs: 1 active+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 activating+undersized+remapped, 1 remapped+peering, 5 peering, 86 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.6 MiB/s wr, 378 op/s; 44/1569 objects degraded (2.804%); 1.1 MiB/s, 58 objects/s recovering 2024-09-17T09:47:17.518 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:17 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 44/1569 objects degraded (2.804%), 1 pg degraded (PG_DEGRADED) 2024-09-17T09:47:17.850 INFO:teuthology.orchestra.run.smithi078.stdout: 5 254007 9.88 MB/sec execute 492 sec latency 54.124 ms 2024-09-17T09:47:17.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:17 smithi136 ceph-mon[49560]: pgmap v224: 97 pgs: 1 active+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 activating+undersized+remapped, 1 remapped+peering, 5 peering, 86 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.6 MiB/s wr, 378 op/s; 44/1569 objects degraded (2.804%); 1.1 MiB/s, 58 objects/s recovering 2024-09-17T09:47:17.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:17 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 44/1569 objects degraded (2.804%), 1 pg degraded (PG_DEGRADED) 2024-09-17T09:47:18.850 INFO:teuthology.orchestra.run.smithi078.stdout: 5 254326 9.88 MB/sec execute 493 sec latency 41.977 ms 2024-09-17T09:47:19.552 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:19 smithi078 ceph-mon[63787]: pgmap v225: 97 pgs: 1 active+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 activating+undersized+remapped, 92 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 423 op/s; 44/1545 objects degraded (2.848%); 892 KiB/s, 34 keys/s, 45 objects/s recovering 2024-09-17T09:47:19.850 INFO:teuthology.orchestra.run.smithi078.stdout: 5 254555 9.86 MB/sec execute 494 sec latency 60.976 ms 2024-09-17T09:47:19.974 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:19 smithi136 ceph-mon[49560]: pgmap v225: 97 pgs: 1 active+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 activating+undersized+remapped, 92 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 423 op/s; 44/1545 objects degraded (2.848%); 892 KiB/s, 34 keys/s, 45 objects/s recovering 2024-09-17T09:47:20.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:20 smithi078 ceph-mon[63787]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 44/1545 objects degraded (2.848%), 1 pg degraded) 2024-09-17T09:47:20.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:20 smithi078 ceph-mon[63787]: Cluster is now healthy 2024-09-17T09:47:20.850 INFO:teuthology.orchestra.run.smithi078.stdout: 5 254808 9.86 MB/sec execute 495 sec latency 55.054 ms 2024-09-17T09:47:20.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:20 smithi136 ceph-mon[49560]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 44/1545 objects degraded (2.848%), 1 pg degraded) 2024-09-17T09:47:20.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:20 smithi136 ceph-mon[49560]: Cluster is now healthy 2024-09-17T09:47:21.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:21 smithi078 ceph-mon[63787]: pgmap v226: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.2 MiB/s wr, 415 op/s; 1.1 MiB/s, 29 keys/s, 40 objects/s recovering 2024-09-17T09:47:21.850 INFO:teuthology.orchestra.run.smithi078.stdout: 5 255073 9.85 MB/sec execute 496 sec latency 55.220 ms 2024-09-17T09:47:21.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:21 smithi136 ceph-mon[49560]: pgmap v226: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.2 MiB/s wr, 415 op/s; 1.1 MiB/s, 29 keys/s, 40 objects/s recovering 2024-09-17T09:47:22.850 INFO:teuthology.orchestra.run.smithi078.stdout: 5 255332 9.85 MB/sec execute 497 sec latency 83.745 ms 2024-09-17T09:47:23.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:23 smithi078 ceph-mon[63787]: pgmap v227: 97 pgs: 97 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.6 MiB/s wr, 392 op/s; 526 KiB/s, 25 keys/s, 20 objects/s recovering 2024-09-17T09:47:23.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:23 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:23.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:23 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:47:23.850 INFO:teuthology.orchestra.run.smithi078.stdout: 5 255563 9.84 MB/sec execute 498 sec latency 87.527 ms 2024-09-17T09:47:23.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:23 smithi136 ceph-mon[49560]: pgmap v227: 97 pgs: 97 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.6 MiB/s wr, 392 op/s; 526 KiB/s, 25 keys/s, 20 objects/s recovering 2024-09-17T09:47:23.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:23 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:23.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:23 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:47:24.850 INFO:teuthology.orchestra.run.smithi078.stdout: 5 255861 9.84 MB/sec execute 499 sec latency 63.078 ms 2024-09-17T09:47:25.718 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:25 smithi078 ceph-mon[63787]: pgmap v228: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.2 MiB/s wr, 394 op/s; 421 KiB/s, 20 keys/s, 16 objects/s recovering 2024-09-17T09:47:25.719 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:25 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-17T09:47:25.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:25 smithi136 ceph-mon[49560]: pgmap v228: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.2 MiB/s wr, 394 op/s; 421 KiB/s, 20 keys/s, 16 objects/s recovering 2024-09-17T09:47:25.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:25 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-17T09:47:25.850 INFO:teuthology.orchestra.run.smithi078.stdout: 5 256128 9.83 MB/sec execute 500 sec latency 58.526 ms 2024-09-17T09:47:26.736 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:26 smithi078 ceph-mon[63787]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T09:47:26.736 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:26 smithi078 ceph-mon[63787]: Upgrade: osd.5 is safe to restart 2024-09-17T09:47:26.736 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:26 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:26.737 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:26 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-17T09:47:26.737 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:26 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:47:26.850 INFO:teuthology.orchestra.run.smithi078.stdout: 5 256531 9.83 MB/sec execute 501 sec latency 44.217 ms 2024-09-17T09:47:26.874 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:26 smithi136 ceph-mon[49560]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T09:47:26.874 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:26 smithi136 ceph-mon[49560]: Upgrade: osd.5 is safe to restart 2024-09-17T09:47:26.875 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:26 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:26.875 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:26 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-17T09:47:26.875 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:26 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:47:27.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:27 smithi136 ceph-mon[49560]: Upgrade: Updating osd.5 2024-09-17T09:47:27.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:27 smithi136 ceph-mon[49560]: Deploying daemon osd.5 on smithi136 2024-09-17T09:47:27.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:27 smithi136 ceph-mon[49560]: pgmap v229: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.9 MiB/s wr, 364 op/s; 383 KiB/s, 18 keys/s, 14 objects/s recovering 2024-09-17T09:47:27.789 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:27 smithi078 ceph-mon[63787]: Upgrade: Updating osd.5 2024-09-17T09:47:27.789 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:27 smithi078 ceph-mon[63787]: Deploying daemon osd.5 on smithi136 2024-09-17T09:47:27.789 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:27 smithi078 ceph-mon[63787]: pgmap v229: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.9 MiB/s wr, 364 op/s; 383 KiB/s, 18 keys/s, 14 objects/s recovering 2024-09-17T09:47:27.850 INFO:teuthology.orchestra.run.smithi078.stdout: 5 256924 9.83 MB/sec execute 502 sec latency 61.156 ms 2024-09-17T09:47:28.850 INFO:teuthology.orchestra.run.smithi078.stdout: 5 257348 9.85 MB/sec execute 503 sec latency 34.853 ms 2024-09-17T09:47:29.047 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:28 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:29.048 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:28 smithi078 ceph-mon[63787]: pgmap v230: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.8 MiB/s wr, 351 op/s; 309 KiB/s, 17 keys/s, 6 objects/s recovering 2024-09-17T09:47:29.183 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:28 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:29.183 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:28 smithi136 ceph-mon[49560]: pgmap v230: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.8 MiB/s wr, 351 op/s; 309 KiB/s, 17 keys/s, 6 objects/s recovering 2024-09-17T09:47:29.850 INFO:teuthology.orchestra.run.smithi078.stdout: 5 257825 9.85 MB/sec execute 504 sec latency 44.369 ms 2024-09-17T09:47:30.349 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:29 smithi136 systemd[1]: Stopping Ceph osd.5 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:47:30.497 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:30 smithi078 ceph-mon[63787]: osd.5 marked itself down and dead 2024-09-17T09:47:30.498 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:30 smithi078 ceph-mon[63787]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:47:30.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:30 smithi136 ceph-mon[49560]: osd.5 marked itself down and dead 2024-09-17T09:47:30.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:30 smithi136 ceph-mon[49560]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:47:30.726 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:30 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[33146]: 2024-09-17T09:47:30.345+0000 7f1c82577700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:47:30.726 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:30 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[33146]: 2024-09-17T09:47:30.345+0000 7f1c82577700 -1 osd.5 132 *** Got signal Terminated *** 2024-09-17T09:47:30.726 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:30 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[33146]: 2024-09-17T09:47:30.346+0000 7f1c82577700 -1 osd.5 132 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:47:30.850 INFO:teuthology.orchestra.run.smithi078.stdout: 5 258259 9.84 MB/sec execute 505 sec latency 424.515 ms 2024-09-17T09:47:31.426 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:31 smithi136 podman[54986]: 2024-09-17 09:47:31.006358927 +0000 UTC m=+0.787136617 container died 30a3049955f928f5e1e08333e305b3a7fd1386b9873664c95c0957d1fab7ee56 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5, io.openshift.expose-services=, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, ceph=True, architecture=x86_64, name=centos-stream, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, release=754, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-09-17T09:47:31.677 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:31 smithi078 ceph-mon[63787]: osdmap e133: 8 total, 7 up, 8 in 2024-09-17T09:47:31.677 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:31 smithi078 ceph-mon[63787]: pgmap v232: 97 pgs: 12 stale+active+clean, 85 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.6 MiB/s wr, 506 op/s 2024-09-17T09:47:31.677 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:31 smithi078 ceph-mon[63787]: osdmap e134: 8 total, 7 up, 8 in 2024-09-17T09:47:31.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:31 smithi136 ceph-mon[49560]: osdmap e133: 8 total, 7 up, 8 in 2024-09-17T09:47:31.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:31 smithi136 ceph-mon[49560]: pgmap v232: 97 pgs: 12 stale+active+clean, 85 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.6 MiB/s wr, 506 op/s 2024-09-17T09:47:31.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:31 smithi136 ceph-mon[49560]: osdmap e134: 8 total, 7 up, 8 in 2024-09-17T09:47:31.850 INFO:teuthology.orchestra.run.smithi078.stdout: 5 258491 9.84 MB/sec execute 506 sec latency 1012.515 ms 2024-09-17T09:47:32.226 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:32 smithi136 podman[54986]: 2024-09-17 09:47:32.020906878 +0000 UTC m=+1.801684565 container cleanup 30a3049955f928f5e1e08333e305b3a7fd1386b9873664c95c0957d1fab7ee56 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, name=centos-stream, vcs-type=git, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, version=8, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, RELEASE=HEAD, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-17T09:47:32.226 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:32 smithi136 bash[54986]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5 2024-09-17T09:47:32.568 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:32 smithi136 podman[55002]: 2024-09-17 09:47:32.289173019 +0000 UTC m=+1.279180035 container remove 30a3049955f928f5e1e08333e305b3a7fd1386b9873664c95c0957d1fab7ee56 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, distribution-scope=public, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, name=centos-stream, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, release=754) 2024-09-17T09:47:32.736 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:32 smithi078 ceph-mon[63787]: Health check failed: Degraded data redundancy: 21/1614 objects degraded (1.301%), 5 pgs degraded (PG_DEGRADED) 2024-09-17T09:47:32.830 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:32 smithi136 ceph-mon[49560]: Health check failed: Degraded data redundancy: 21/1614 objects degraded (1.301%), 5 pgs degraded (PG_DEGRADED) 2024-09-17T09:47:32.830 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:32 smithi136 podman[55087]: 2024-09-17 09:47:32.555049263 +0000 UTC m=+0.019682641 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:47:32.830 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:32 smithi136 podman[55087]: 2024-09-17 09:47:32.681869028 +0000 UTC m=+0.146502396 container create 746d82e5e3b87deb47c1bd7a1b33ca7c1a21a81929d1885866d657748885d22f (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, CEPH_POINT_RELEASE=) 2024-09-17T09:47:32.850 INFO:teuthology.orchestra.run.smithi078.stdout: 5 258944 9.85 MB/sec execute 507 sec latency 42.200 ms 2024-09-17T09:47:33.226 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:32 smithi136 podman[55087]: 2024-09-17 09:47:32.959753192 +0000 UTC m=+0.424386569 container init 746d82e5e3b87deb47c1bd7a1b33ca7c1a21a81929d1885866d657748885d22f (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5-deactivate, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-17T09:47:33.226 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:32 smithi136 podman[55087]: 2024-09-17 09:47:32.964244765 +0000 UTC m=+0.428878137 container start 746d82e5e3b87deb47c1bd7a1b33ca7c1a21a81929d1885866d657748885d22f (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-17T09:47:33.226 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:33 smithi136 podman[55087]: 2024-09-17 09:47:33.060175917 +0000 UTC m=+0.524809295 container attach 746d82e5e3b87deb47c1bd7a1b33ca7c1a21a81929d1885866d657748885d22f (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-17T09:47:33.226 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:33 smithi136 podman[55087]: 2024-09-17 09:47:33.197742511 +0000 UTC m=+0.662375886 container died 746d82e5e3b87deb47c1bd7a1b33ca7c1a21a81929d1885866d657748885d22f (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:47:33.764 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:33 smithi078 ceph-mon[63787]: pgmap v234: 97 pgs: 2 active+undersized, 5 peering, 7 stale+active+clean, 5 active+undersized+degraded, 78 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 4.1 MiB/s wr, 474 op/s; 21/1614 objects degraded (1.301%) 2024-09-17T09:47:33.850 INFO:teuthology.orchestra.run.smithi078.stdout: 5 259364 9.85 MB/sec execute 508 sec latency 75.741 ms 2024-09-17T09:47:33.890 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:33 smithi136 ceph-mon[49560]: pgmap v234: 97 pgs: 2 active+undersized, 5 peering, 7 stale+active+clean, 5 active+undersized+degraded, 78 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 4.1 MiB/s wr, 474 op/s; 21/1614 objects degraded (1.301%) 2024-09-17T09:47:34.141 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:33 smithi136 podman[55111]: 2024-09-17 09:47:33.889609897 +0000 UTC m=+0.680939138 container remove 746d82e5e3b87deb47c1bd7a1b33ca7c1a21a81929d1885866d657748885d22f (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-17T09:47:34.141 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:33 smithi136 systemd[1]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.5.service: Deactivated successfully. 2024-09-17T09:47:34.141 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:33 smithi136 systemd[1]: Stopped Ceph osd.5 for 35c0cd22-74d7-11ef-bceb-c7b262605968. 2024-09-17T09:47:34.141 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:33 smithi136 systemd[1]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.5.service: Consumed 5min 16.453s CPU time. 2024-09-17T09:47:34.142 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:34 smithi136 systemd[1]: Starting Ceph osd.5 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:47:34.658 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:34 smithi136 ceph-mon[49560]: pgmap v235: 97 pgs: 6 active+undersized, 5 peering, 20 active+undersized+degraded, 66 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.1 MiB/s wr, 577 op/s; 151/1656 objects degraded (9.118%) 2024-09-17T09:47:34.659 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:34 smithi136 podman[55209]: 2024-09-17 09:47:34.280475518 +0000 UTC m=+0.018505899 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:47:34.659 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:34 smithi136 podman[55209]: 2024-09-17 09:47:34.657858603 +0000 UTC m=+0.395888975 container create de8cd2ec413f8e5285cd489a8800d11352517d30badbde94211fb2a434a2bb9e (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, ceph=True) 2024-09-17T09:47:34.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:34 smithi078 ceph-mon[63787]: pgmap v235: 97 pgs: 6 active+undersized, 5 peering, 20 active+undersized+degraded, 66 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.1 MiB/s wr, 577 op/s; 151/1656 objects degraded (9.118%) 2024-09-17T09:47:34.850 INFO:teuthology.orchestra.run.smithi078.stdout: 5 259757 9.84 MB/sec execute 509 sec latency 42.911 ms 2024-09-17T09:47:35.226 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:34 smithi136 podman[55209]: 2024-09-17 09:47:34.927649891 +0000 UTC m=+0.665680269 container init de8cd2ec413f8e5285cd489a8800d11352517d30badbde94211fb2a434a2bb9e (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-17T09:47:35.226 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:34 smithi136 podman[55209]: 2024-09-17 09:47:34.931512133 +0000 UTC m=+0.669542511 container start de8cd2ec413f8e5285cd489a8800d11352517d30badbde94211fb2a434a2bb9e (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5-activate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-17T09:47:35.226 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:35 smithi136 podman[55209]: 2024-09-17 09:47:35.036233691 +0000 UTC m=+0.774264080 container attach de8cd2ec413f8e5285cd489a8800d11352517d30badbde94211fb2a434a2bb9e (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, io.buildah.version=1.37.2) 2024-09-17T09:47:35.226 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:35 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5-activate[55224]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:47:35.226 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:35 smithi136 bash[55209]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:47:35.226 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:35 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5-activate[55224]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:47:35.226 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:35 smithi136 bash[55209]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:47:35.818 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:35 smithi078 ceph-mon[63787]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-17T09:47:35.850 INFO:teuthology.orchestra.run.smithi078.stdout: 5 260086 9.85 MB/sec execute 510 sec latency 46.414 ms 2024-09-17T09:47:35.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:35 smithi136 ceph-mon[49560]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-17T09:47:36.476 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:36 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5-activate[55224]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-17T09:47:36.476 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:36 smithi136 bash[55209]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-17T09:47:36.476 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:36 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5-activate[55224]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-17T09:47:36.476 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:36 smithi136 bash[55209]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-17T09:47:36.830 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:36 smithi078 ceph-mon[63787]: pgmap v236: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.9 MiB/s wr, 595 op/s; 169/1623 objects degraded (10.413%) 2024-09-17T09:47:36.850 INFO:teuthology.orchestra.run.smithi078.stdout: 5 260504 9.85 MB/sec execute 511 sec latency 51.843 ms 2024-09-17T09:47:36.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:36 smithi136 ceph-mon[49560]: pgmap v236: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.9 MiB/s wr, 595 op/s; 169/1623 objects degraded (10.413%) 2024-09-17T09:47:36.976 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:36 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5-activate[55224]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-17T09:47:36.976 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:36 smithi136 bash[55209]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-17T09:47:36.976 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:36 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5-activate[55224]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-17T09:47:36.976 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:36 smithi136 bash[55209]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-17T09:47:36.976 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:36 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5-activate[55224]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-17T09:47:36.976 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:36 smithi136 bash[55209]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-17T09:47:36.976 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:36 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5-activate[55224]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-17T09:47:36.977 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:36 smithi136 bash[55209]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-17T09:47:36.977 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:36 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5-activate[55224]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-17T09:47:36.977 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:36 smithi136 bash[55209]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-17T09:47:36.977 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:36 smithi136 podman[55209]: 2024-09-17 09:47:36.520142398 +0000 UTC m=+2.258172773 container died de8cd2ec413f8e5285cd489a8800d11352517d30badbde94211fb2a434a2bb9e (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5-activate, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T09:47:37.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:37 smithi078 ceph-mon[63787]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-17T09:47:37.850 INFO:teuthology.orchestra.run.smithi078.stdout: 5 260828 9.86 MB/sec execute 512 sec latency 40.390 ms 2024-09-17T09:47:37.958 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:37 smithi136 ceph-mon[49560]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-17T09:47:38.225 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:37 smithi136 podman[55356]: 2024-09-17 09:47:37.957847549 +0000 UTC m=+1.427013872 container remove de8cd2ec413f8e5285cd489a8800d11352517d30badbde94211fb2a434a2bb9e (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T09:47:38.226 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:38 smithi136 podman[55412]: 2024-09-17 09:47:38.082413173 +0000 UTC m=+0.019677497 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:47:38.473 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-17T09:47:38.479 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:38 smithi136 podman[55412]: 2024-09-17 09:47:38.225930865 +0000 UTC m=+0.163195194 container create df9b6954ce0f7365eb4b5ee5f2940b7c9bc6589537b8702945231786d886afbc (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-09-17T09:47:38.850 INFO:teuthology.orchestra.run.smithi078.stdout: 5 261159 9.86 MB/sec execute 513 sec latency 44.277 ms 2024-09-17T09:47:38.876 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:38.876 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:47:38.877 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:38 smithi078 ceph-mon[63787]: from='client.34378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:47:38.877 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:38 smithi078 ceph-mon[63787]: pgmap v237: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 5.8 MiB/s wr, 571 op/s; 180/1686 objects degraded (10.676%) 2024-09-17T09:47:38.877 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:38.877 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:38.877 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:47:38.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:38.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:47:38.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:38 smithi136 ceph-mon[49560]: from='client.34378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:47:38.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:38 smithi136 ceph-mon[49560]: pgmap v237: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 5.8 MiB/s wr, 571 op/s; 180/1686 objects degraded (10.676%) 2024-09-17T09:47:38.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:38.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:38.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:47:38.976 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:38 smithi136 podman[55412]: 2024-09-17 09:47:38.478726525 +0000 UTC m=+0.415990864 container init df9b6954ce0f7365eb4b5ee5f2940b7c9bc6589537b8702945231786d886afbc (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T09:47:38.976 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:38 smithi136 podman[55412]: 2024-09-17 09:47:38.482563427 +0000 UTC m=+0.419827757 container start df9b6954ce0f7365eb4b5ee5f2940b7c9bc6589537b8702945231786d886afbc (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-17T09:47:38.976 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:38 smithi136 ceph-osd[55430]: -- 172.21.15.136:0/409978916 <== mon.1 v2:172.21.15.136:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5600e20421e0 con 0x5600e11ddc00 2024-09-17T09:47:38.976 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:38 smithi136 bash[55412]: df9b6954ce0f7365eb4b5ee5f2940b7c9bc6589537b8702945231786d886afbc 2024-09-17T09:47:38.976 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:38 smithi136 systemd[1]: Started Ceph osd.5 for 35c0cd22-74d7-11ef-bceb-c7b262605968. 2024-09-17T09:47:39.496 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:47:39.496 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (8m) 73s ago 17m 17.8M - 0.27.0 11f11916f8cd 97db2373ce92 2024-09-17T09:47:39.496 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (4m) 73s ago 17m 6899k - 19.3.0-4963-ge646f729 f271290a0950 da528c5ca445 2024-09-17T09:47:39.496 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi136 smithi136 running (4m) 34s ago 16m 6899k - 19.3.0-4963-ge646f729 f271290a0950 909f405348cc 2024-09-17T09:47:39.496 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (7m) 73s ago 16m 53.0M - dad864ee21e9 db74b0f8e0a7 2024-09-17T09:47:39.496 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.jdrsup smithi078 running (11m) 73s ago 11m 38.4M - 17.2.0 e1d6a67b021e 8f834e15ba88 2024-09-17T09:47:39.496 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi136.smslff smithi136 running (11m) 34s ago 11m 16.5M - 17.2.0 e1d6a67b021e 0c20e2b328d3 2024-09-17T09:47:39.496 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rsqofi smithi078 *:8443,9283,8765 running (7m) 73s ago 18m 519M - 19.3.0-4963-ge646f729 f271290a0950 7dda3108c9a0 2024-09-17T09:47:39.496 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi136.hicmlm smithi136 *:8443,9283,8765 running (6m) 34s ago 16m 456M - 19.3.0-4963-ge646f729 f271290a0950 fc518b58e07a 2024-09-17T09:47:39.497 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (5m) 73s ago 18m 53.2M 2048M 19.3.0-4963-ge646f729 f271290a0950 3ead101d3db1 2024-09-17T09:47:39.497 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi136 smithi136 running (5m) 34s ago 16m 42.2M 2048M 19.3.0-4963-ge646f729 f271290a0950 4fce88842b79 2024-09-17T09:47:39.497 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.tcwbvi smithi078 *:2049 running (10m) 73s ago 10m 284M - 4.0 e1d6a67b021e 30c87cb46555 2024-09-17T09:47:39.497 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi136.regjvc smithi136 *:2049 running (10m) 34s ago 10m 90.2M - 4.0 e1d6a67b021e b88e4dc24376 2024-09-17T09:47:39.497 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (7m) 73s ago 16m 11.0M - 1.7.0 72c9c2088986 52dad3adeeaa 2024-09-17T09:47:39.497 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi136 smithi136 *:9100 running (7m) 34s ago 16m 11.8M - 1.7.0 72c9c2088986 f2bc5856e35a 2024-09-17T09:47:39.497 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (3m) 73s ago 15m 351M 1936M 19.3.0-4963-ge646f729 f271290a0950 1ea81cc809b8 2024-09-17T09:47:39.497 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (3m) 73s ago 15m 290M 1936M 19.3.0-4963-ge646f729 f271290a0950 eda32560334b 2024-09-17T09:47:39.497 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (2m) 73s ago 14m 213M 1936M 19.3.0-4963-ge646f729 f271290a0950 83a32ad7c358 2024-09-17T09:47:39.497 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (84s) 73s ago 14m 14.3M 1936M 19.3.0-4963-ge646f729 f271290a0950 e2cd18998d2b 2024-09-17T09:47:39.497 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi136 running (38s) 34s ago 13m 11.6M 2704M 19.3.0-4963-ge646f729 f271290a0950 e2a18ad94664 2024-09-17T09:47:39.497 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi136 starting - - - 2704M 2024-09-17T09:47:39.497 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi136 running (13m) 34s ago 13m 530M 2704M 17.2.0 e1d6a67b021e 1698e1567c33 2024-09-17T09:47:39.498 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi136 running (12m) 34s ago 12m 796M 2704M 17.2.0 e1d6a67b021e 5d3b08cecd78 2024-09-17T09:47:39.498 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (6m) 73s ago 16m 53.8M - 2.51.0 1d3b7f56885b 6e5e8c721eb0 2024-09-17T09:47:39.850 INFO:teuthology.orchestra.run.smithi078.stdout: 5 261453 9.84 MB/sec execute 514 sec latency 52.686 ms 2024-09-17T09:47:39.892 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:39 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 180/1686 objects degraded (10.676%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T09:47:39.892 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:39 smithi078 ceph-mon[63787]: from='client.34384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:47:39.892 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:39 smithi078 ceph-mon[63787]: from='client.34388 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:47:39.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:39 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 180/1686 objects degraded (10.676%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T09:47:39.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:39 smithi136 ceph-mon[49560]: from='client.34384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:47:39.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:39 smithi136 ceph-mon[49560]: from='client.34388 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:47:40.017 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:47:40.017 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-17T09:47:40.017 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:47:40.017 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:47:40.017 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-17T09:47:40.017 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:47:40.017 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:47:40.017 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-17T09:47:40.018 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-17T09:47:40.018 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 5 2024-09-17T09:47:40.018 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:47:40.018 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-17T09:47:40.018 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:47:40.018 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:47:40.018 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-17T09:47:40.018 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-17T09:47:40.018 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 9 2024-09-17T09:47:40.018 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-17T09:47:40.018 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:47:40.475 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:40 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:47:40.360+0000 7f909765a640 -1 Falling back to public interface 2024-09-17T09:47:40.511 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:47:40.511 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407", 2024-09-17T09:47:40.511 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-17T09:47:40.511 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T09:47:40.511 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-17T09:47:40.512 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-09-17T09:47:40.512 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-17T09:47:40.512 INFO:teuthology.orchestra.run.smithi078.stdout: "mon" 2024-09-17T09:47:40.512 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-17T09:47:40.512 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "11/23 daemons upgraded", 2024-09-17T09:47:40.512 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T09:47:40.512 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-17T09:47:40.512 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:47:40.850 INFO:teuthology.orchestra.run.smithi078.stdout: 5 261705 9.83 MB/sec execute 515 sec latency 59.630 ms 2024-09-17T09:47:40.967 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:40 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/742911829' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:47:40.967 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:40 smithi078 ceph-mon[63787]: pgmap v238: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.5 MiB/s wr, 510 op/s; 179/1644 objects degraded (10.888%) 2024-09-17T09:47:40.967 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:40 smithi078 ceph-mon[63787]: from='client.34396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:47:41.016 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:40 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/742911829' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:47:41.016 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:40 smithi136 ceph-mon[49560]: pgmap v238: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.5 MiB/s wr, 510 op/s; 179/1644 objects degraded (10.888%) 2024-09-17T09:47:41.016 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:40 smithi136 ceph-mon[49560]: from='client.34396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:47:41.033 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 179/1644 objects degraded (10.888%), 23 pgs degraded 2024-09-17T09:47:41.033 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-17T09:47:41.033 INFO:teuthology.orchestra.run.smithi078.stdout: osd.5 (root=default,host=smithi136) is down 2024-09-17T09:47:41.034 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 179/1644 objects degraded (10.888%), 23 pgs degraded 2024-09-17T09:47:41.034 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.2 is active+undersized+degraded, acting [1,6] 2024-09-17T09:47:41.034 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.3 is active+undersized+degraded, acting [2,4] 2024-09-17T09:47:41.034 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.b is active+undersized+degraded, acting [7,4] 2024-09-17T09:47:41.034 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.c is active+undersized+degraded, acting [2,0] 2024-09-17T09:47:41.034 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.12 is active+undersized+degraded, acting [3,7] 2024-09-17T09:47:41.034 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.14 is active+undersized+degraded, acting [6,3] 2024-09-17T09:47:41.034 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.15 is active+undersized+degraded, acting [1,0] 2024-09-17T09:47:41.034 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.16 is active+undersized+degraded, acting [6,2] 2024-09-17T09:47:41.034 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.17 is active+undersized+degraded, acting [6,2] 2024-09-17T09:47:41.034 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.18 is active+undersized+degraded, acting [4,7] 2024-09-17T09:47:41.034 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1c is active+undersized+degraded, acting [7,2] 2024-09-17T09:47:41.034 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1e is active+undersized+degraded, acting [3,0] 2024-09-17T09:47:41.035 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.2 is active+undersized+degraded, acting [3,6] 2024-09-17T09:47:41.035 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.4 is active+undersized+degraded, acting [1,2] 2024-09-17T09:47:41.035 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.5 is active+undersized+degraded, acting [3,2] 2024-09-17T09:47:41.035 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.c is active+undersized+degraded, acting [3,6] 2024-09-17T09:47:41.035 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.d is active+undersized+degraded, acting [7,6] 2024-09-17T09:47:41.035 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.10 is active+undersized+degraded, acting [6,0] 2024-09-17T09:47:41.035 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.16 is active+undersized+degraded, acting [7,1] 2024-09-17T09:47:41.035 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.17 is active+undersized+degraded, acting [0,3] 2024-09-17T09:47:41.035 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1c is active+undersized+degraded, acting [4,1] 2024-09-17T09:47:41.035 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1d is active+undersized+degraded, acting [4,6] 2024-09-17T09:47:41.035 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1f is active+undersized+degraded, acting [0,2] 2024-09-17T09:47:41.741 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:41 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/798077122' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:47:41.850 INFO:teuthology.orchestra.run.smithi078.stdout: 5 261909 9.82 MB/sec execute 516 sec latency 65.366 ms 2024-09-17T09:47:42.020 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:41 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/798077122' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:47:42.850 INFO:teuthology.orchestra.run.smithi078.stdout: 5 262255 9.83 MB/sec execute 517 sec latency 62.427 ms 2024-09-17T09:47:43.537 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:43 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:43.537 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:43 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:43.538 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:43 smithi078 ceph-mon[63787]: pgmap v239: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.0 MiB/s wr, 431 op/s; 181/1614 objects degraded (11.214%) 2024-09-17T09:47:43.538 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:43 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:43.538 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:43 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:43.634 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:43 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:43.634 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:43 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:43.634 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:43 smithi136 ceph-mon[49560]: pgmap v239: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.0 MiB/s wr, 431 op/s; 181/1614 objects degraded (11.214%) 2024-09-17T09:47:43.634 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:43 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:43.634 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:43 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:43.851 INFO:teuthology.orchestra.run.smithi078.stdout: 5 262591 9.83 MB/sec execute 518 sec latency 49.474 ms 2024-09-17T09:47:44.803 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:44 smithi136 ceph-mon[49560]: pgmap v240: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.9 MiB/s wr, 565 op/s; 170/1554 objects degraded (10.940%) 2024-09-17T09:47:44.803 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:44 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 170/1554 objects degraded (10.940%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T09:47:44.828 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:44 smithi078 ceph-mon[63787]: pgmap v240: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.9 MiB/s wr, 565 op/s; 170/1554 objects degraded (10.940%) 2024-09-17T09:47:44.828 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:44 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 170/1554 objects degraded (10.940%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T09:47:44.850 INFO:teuthology.orchestra.run.smithi078.stdout: 5 262974 9.83 MB/sec execute 519 sec latency 55.529 ms 2024-09-17T09:47:45.851 INFO:teuthology.orchestra.run.smithi078.stdout: 5 263270 9.82 MB/sec execute 520 sec latency 47.334 ms 2024-09-17T09:47:46.851 INFO:teuthology.orchestra.run.smithi078.stdout: 5 263503 9.81 MB/sec execute 521 sec latency 44.986 ms 2024-09-17T09:47:47.673 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:47 smithi078 ceph-mon[63787]: pgmap v241: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.6 MiB/s wr, 503 op/s; 170/1560 objects degraded (10.897%) 2024-09-17T09:47:47.673 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:47 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:47.673 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:47 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:47.673 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:47 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:47:47.673 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:47 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:47:47.673 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:47 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:47.673 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:47 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:47.673 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:47 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:47:47.673 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:47 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:47:47.674 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:47 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:47:47.674 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:47 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:47:47.674 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:47 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-17T09:47:47.851 INFO:teuthology.orchestra.run.smithi078.stdout: 5 263928 9.82 MB/sec execute 522 sec latency 49.598 ms 2024-09-17T09:47:47.877 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:47 smithi136 ceph-mon[49560]: pgmap v241: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.6 MiB/s wr, 503 op/s; 170/1560 objects degraded (10.897%) 2024-09-17T09:47:47.877 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:47 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:47.877 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:47 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:47.877 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:47 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:47:47.877 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:47 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:47:47.877 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:47 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:47.878 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:47 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:47.878 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:47 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:47:47.878 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:47 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:47:47.878 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:47 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:47:47.878 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:47 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:47:47.878 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:47 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-17T09:47:48.701 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:48 smithi078 ceph-mon[63787]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T09:47:48.701 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:48 smithi078 ceph-mon[63787]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-17T09:47:48.725 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:48 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:47:48.446+0000 7f909765a640 -1 osd.5 0 read_superblock omap replica is missing. 2024-09-17T09:47:48.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:48 smithi136 ceph-mon[49560]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T09:47:48.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:48 smithi136 ceph-mon[49560]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-17T09:47:48.851 INFO:teuthology.orchestra.run.smithi078.stdout: 5 264239 9.82 MB/sec execute 523 sec latency 69.542 ms 2024-09-17T09:47:49.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:49 smithi136 ceph-mon[49560]: pgmap v242: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.1 MiB/s wr, 541 op/s; 168/1584 objects degraded (10.606%) 2024-09-17T09:47:49.755 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:49 smithi078 ceph-mon[63787]: pgmap v242: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.1 MiB/s wr, 541 op/s; 168/1584 objects degraded (10.606%) 2024-09-17T09:47:49.851 INFO:teuthology.orchestra.run.smithi078.stdout: 5 264505 9.82 MB/sec execute 524 sec latency 63.508 ms 2024-09-17T09:47:50.225 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:49 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:47:49.760+0000 7f909765a640 -1 osd.5 132 log_to_monitors true 2024-09-17T09:47:50.798 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:50 smithi078 ceph-mon[63787]: from='osd.5 [v2:172.21.15.136:6808/2971512918,v1:172.21.15.136:6809/2971512918]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-17T09:47:50.798 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:50 smithi078 ceph-mon[63787]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-17T09:47:50.798 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:50 smithi078 ceph-mon[63787]: pgmap v243: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.5 MiB/s wr, 427 op/s; 173/1635 objects degraded (10.581%) 2024-09-17T09:47:50.851 INFO:teuthology.orchestra.run.smithi078.stdout: 5 264722 9.82 MB/sec execute 525 sec latency 88.818 ms 2024-09-17T09:47:50.904 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:50 smithi136 ceph-mon[49560]: from='osd.5 [v2:172.21.15.136:6808/2971512918,v1:172.21.15.136:6809/2971512918]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-17T09:47:50.904 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:50 smithi136 ceph-mon[49560]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-17T09:47:50.904 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:50 smithi136 ceph-mon[49560]: pgmap v243: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.5 MiB/s wr, 427 op/s; 173/1635 objects degraded (10.581%) 2024-09-17T09:47:51.225 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:47:50 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:47:50.902+0000 7f908b3fc640 -1 osd.5 132 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T09:47:51.805 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:51 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 173/1635 objects degraded (10.581%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T09:47:51.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:51 smithi078 ceph-mon[63787]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-17T09:47:51.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:51 smithi078 ceph-mon[63787]: from='osd.5 [v2:172.21.15.136:6808/2971512918,v1:172.21.15.136:6809/2971512918]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi136", "root=default"]} : dispatch 2024-09-17T09:47:51.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:51 smithi078 ceph-mon[63787]: osdmap e135: 8 total, 7 up, 8 in 2024-09-17T09:47:51.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:51 smithi078 ceph-mon[63787]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi136", "root=default"]} : dispatch 2024-09-17T09:47:51.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:51 smithi078 ceph-mon[63787]: from='osd.5 ' entity='osd.5' 2024-09-17T09:47:51.851 INFO:teuthology.orchestra.run.smithi078.stdout: 5 265013 9.80 MB/sec execute 526 sec latency 64.604 ms 2024-09-17T09:47:51.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:51 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 173/1635 objects degraded (10.581%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T09:47:51.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:51 smithi136 ceph-mon[49560]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-17T09:47:51.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:51 smithi136 ceph-mon[49560]: from='osd.5 [v2:172.21.15.136:6808/2971512918,v1:172.21.15.136:6809/2971512918]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi136", "root=default"]} : dispatch 2024-09-17T09:47:51.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:51 smithi136 ceph-mon[49560]: osdmap e135: 8 total, 7 up, 8 in 2024-09-17T09:47:51.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:51 smithi136 ceph-mon[49560]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi136", "root=default"]} : dispatch 2024-09-17T09:47:51.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:51 smithi136 ceph-mon[49560]: from='osd.5 ' entity='osd.5' 2024-09-17T09:47:52.828 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:52 smithi078 ceph-mon[63787]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:47:52.829 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:52 smithi078 ceph-mon[63787]: osd.5 [v2:172.21.15.136:6808/2971512918,v1:172.21.15.136:6809/2971512918] boot 2024-09-17T09:47:52.829 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:52 smithi078 ceph-mon[63787]: osdmap e136: 8 total, 8 up, 8 in 2024-09-17T09:47:52.829 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-17T09:47:52.829 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:52 smithi078 ceph-mon[63787]: osdmap e137: 8 total, 8 up, 8 in 2024-09-17T09:47:52.829 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:52 smithi078 ceph-mon[63787]: pgmap v247: 97 pgs: 5 peering, 6 active+undersized, 20 active+undersized+degraded, 66 active+clean; 349 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 4.4 MiB/s wr, 481 op/s; 154/1638 objects degraded (9.402%) 2024-09-17T09:47:52.851 INFO:teuthology.orchestra.run.smithi078.stdout: 5 265204 9.79 MB/sec execute 527 sec latency 900.452 ms 2024-09-17T09:47:52.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:52 smithi136 ceph-mon[49560]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:47:52.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:52 smithi136 ceph-mon[49560]: osd.5 [v2:172.21.15.136:6808/2971512918,v1:172.21.15.136:6809/2971512918] boot 2024-09-17T09:47:52.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:52 smithi136 ceph-mon[49560]: osdmap e136: 8 total, 8 up, 8 in 2024-09-17T09:47:52.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:52 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-17T09:47:52.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:52 smithi136 ceph-mon[49560]: osdmap e137: 8 total, 8 up, 8 in 2024-09-17T09:47:52.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:52 smithi136 ceph-mon[49560]: pgmap v247: 97 pgs: 5 peering, 6 active+undersized, 20 active+undersized+degraded, 66 active+clean; 349 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 4.4 MiB/s wr, 481 op/s; 154/1638 objects degraded (9.402%) 2024-09-17T09:47:53.804 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:53 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:53.805 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:53 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:47:53.805 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:53 smithi078 ceph-mon[63787]: osdmap e138: 8 total, 8 up, 8 in 2024-09-17T09:47:53.851 INFO:teuthology.orchestra.run.smithi078.stdout: 5 265332 9.77 MB/sec execute 528 sec latency 1575.113 ms 2024-09-17T09:47:53.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:53 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:47:53.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:53 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:47:53.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:53 smithi136 ceph-mon[49560]: osdmap e138: 8 total, 8 up, 8 in 2024-09-17T09:47:54.851 INFO:teuthology.orchestra.run.smithi078.stdout: 5 265498 9.76 MB/sec execute 529 sec latency 328.912 ms 2024-09-17T09:47:55.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:55 smithi078 ceph-mon[63787]: pgmap v250: 97 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 activating+undersized+degraded+remapped, 2 remapped+peering, 5 peering, 4 active+undersized, 9 active+undersized+degraded, 73 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.5 MiB/s wr, 481 op/s; 121/1608 objects degraded (7.525%); 247 KiB/s, 9 objects/s recovering 2024-09-17T09:47:55.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:55 smithi078 ceph-mon[63787]: osdmap e139: 8 total, 8 up, 8 in 2024-09-17T09:47:55.728 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:55 smithi136 ceph-mon[49560]: pgmap v250: 97 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 activating+undersized+degraded+remapped, 2 remapped+peering, 5 peering, 4 active+undersized, 9 active+undersized+degraded, 73 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.5 MiB/s wr, 481 op/s; 121/1608 objects degraded (7.525%); 247 KiB/s, 9 objects/s recovering 2024-09-17T09:47:55.728 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:55 smithi136 ceph-mon[49560]: osdmap e139: 8 total, 8 up, 8 in 2024-09-17T09:47:55.851 INFO:teuthology.orchestra.run.smithi078.stdout: 5 265682 9.75 MB/sec execute 530 sec latency 981.627 ms 2024-09-17T09:47:56.639 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:56 smithi078 ceph-mon[63787]: osdmap e140: 8 total, 8 up, 8 in 2024-09-17T09:47:56.851 INFO:teuthology.orchestra.run.smithi078.stdout: 5 265935 9.74 MB/sec execute 531 sec latency 822.912 ms 2024-09-17T09:47:56.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:56 smithi136 ceph-mon[49560]: osdmap e140: 8 total, 8 up, 8 in 2024-09-17T09:47:57.667 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:57 smithi078 ceph-mon[63787]: pgmap v252: 97 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 remapped+peering, 7 peering, 85 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 3.1 MiB/s wr, 256 op/s; 54/1551 objects degraded (3.482%); 3.7 MiB/s, 37 objects/s recovering 2024-09-17T09:47:57.668 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:57 smithi078 ceph-mon[63787]: osdmap e141: 8 total, 8 up, 8 in 2024-09-17T09:47:57.668 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:57 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 121/1608 objects degraded (7.525%), 11 pgs degraded (PG_DEGRADED) 2024-09-17T09:47:57.851 INFO:teuthology.orchestra.run.smithi078.stdout: 5 266373 9.75 MB/sec execute 532 sec latency 44.880 ms 2024-09-17T09:47:57.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:57 smithi136 ceph-mon[49560]: pgmap v252: 97 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 activating+undersized+degraded+remapped, 1 remapped+peering, 7 peering, 85 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 3.1 MiB/s wr, 256 op/s; 54/1551 objects degraded (3.482%); 3.7 MiB/s, 37 objects/s recovering 2024-09-17T09:47:57.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:57 smithi136 ceph-mon[49560]: osdmap e141: 8 total, 8 up, 8 in 2024-09-17T09:47:57.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:57 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 121/1608 objects degraded (7.525%), 11 pgs degraded (PG_DEGRADED) 2024-09-17T09:47:58.851 INFO:teuthology.orchestra.run.smithi078.stdout: 5 266808 9.76 MB/sec execute 533 sec latency 42.520 ms 2024-09-17T09:47:59.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:47:59 smithi078 ceph-mon[63787]: pgmap v254: 97 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 activating+undersized+degraded+remapped, 2 peering, 91 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 2.2 MiB/s wr, 147 op/s; 54/1539 objects degraded (3.509%); 3.2 MiB/s, 40 objects/s recovering 2024-09-17T09:47:59.755 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:47:59 smithi136 ceph-mon[49560]: pgmap v254: 97 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+remapped, 1 activating+undersized+degraded+remapped, 2 peering, 91 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 2.2 MiB/s wr, 147 op/s; 54/1539 objects degraded (3.509%); 3.2 MiB/s, 40 objects/s recovering 2024-09-17T09:47:59.851 INFO:teuthology.orchestra.run.smithi078.stdout: 5 267079 9.76 MB/sec execute 534 sec latency 50.892 ms 2024-09-17T09:48:00.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:00 smithi136 ceph-mon[49560]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 54/1539 objects degraded (3.509%), 2 pgs degraded) 2024-09-17T09:48:00.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:00 smithi136 ceph-mon[49560]: Cluster is now healthy 2024-09-17T09:48:00.764 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:00 smithi078 ceph-mon[63787]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 54/1539 objects degraded (3.509%), 2 pgs degraded) 2024-09-17T09:48:00.764 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:00 smithi078 ceph-mon[63787]: Cluster is now healthy 2024-09-17T09:48:00.851 INFO:teuthology.orchestra.run.smithi078.stdout: 5 267400 9.76 MB/sec execute 535 sec latency 40.136 ms 2024-09-17T09:48:01.774 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:01 smithi078 ceph-mon[63787]: pgmap v255: 97 pgs: 2 peering, 95 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.2 MiB/s wr, 348 op/s; 3.2 MiB/s, 36 objects/s recovering 2024-09-17T09:48:01.851 INFO:teuthology.orchestra.run.smithi078.stdout: 5 267713 9.76 MB/sec execute 536 sec latency 54.015 ms 2024-09-17T09:48:01.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:01 smithi136 ceph-mon[49560]: pgmap v255: 97 pgs: 2 peering, 95 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.2 MiB/s wr, 348 op/s; 3.2 MiB/s, 36 objects/s recovering 2024-09-17T09:48:02.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:02 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-17T09:48:02.851 INFO:teuthology.orchestra.run.smithi078.stdout: 5 268010 9.76 MB/sec execute 537 sec latency 57.803 ms 2024-09-17T09:48:02.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:02 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-17T09:48:03.568 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:03 smithi078 ceph-mon[63787]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T09:48:03.569 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:03 smithi078 ceph-mon[63787]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-17T09:48:03.569 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:03 smithi078 ceph-mon[63787]: pgmap v256: 97 pgs: 97 active+clean; 346 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.8 MiB/s wr, 429 op/s; 2.4 MiB/s, 27 objects/s recovering 2024-09-17T09:48:03.851 INFO:teuthology.orchestra.run.smithi078.stdout: 5 268259 9.75 MB/sec execute 538 sec latency 67.356 ms 2024-09-17T09:48:03.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:03 smithi136 ceph-mon[49560]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T09:48:03.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:03 smithi136 ceph-mon[49560]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-17T09:48:03.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:03 smithi136 ceph-mon[49560]: pgmap v256: 97 pgs: 97 active+clean; 346 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.8 MiB/s wr, 429 op/s; 2.4 MiB/s, 27 objects/s recovering 2024-09-17T09:48:04.851 INFO:teuthology.orchestra.run.smithi078.stdout: 5 268572 9.75 MB/sec execute 539 sec latency 75.271 ms 2024-09-17T09:48:05.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:05 smithi078 ceph-mon[63787]: pgmap v257: 97 pgs: 97 active+clean; 346 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.0 MiB/s wr, 502 op/s; 2.1 MiB/s, 24 objects/s recovering 2024-09-17T09:48:05.851 INFO:teuthology.orchestra.run.smithi078.stdout: 5 268972 9.74 MB/sec execute 540 sec latency 41.264 ms 2024-09-17T09:48:05.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:05 smithi136 ceph-mon[49560]: pgmap v257: 97 pgs: 97 active+clean; 346 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.0 MiB/s wr, 502 op/s; 2.1 MiB/s, 24 objects/s recovering 2024-09-17T09:48:06.851 INFO:teuthology.orchestra.run.smithi078.stdout: 5 269236 9.73 MB/sec execute 541 sec latency 63.066 ms 2024-09-17T09:48:07.663 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:07 smithi078 ceph-mon[63787]: pgmap v258: 97 pgs: 97 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 455 op/s; 517 KiB/s, 10 objects/s recovering 2024-09-17T09:48:07.851 INFO:teuthology.orchestra.run.smithi078.stdout: 5 269548 9.73 MB/sec execute 542 sec latency 60.816 ms 2024-09-17T09:48:07.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:07 smithi136 ceph-mon[49560]: pgmap v258: 97 pgs: 97 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 455 op/s; 517 KiB/s, 10 objects/s recovering 2024-09-17T09:48:08.851 INFO:teuthology.orchestra.run.smithi078.stdout: 5 269948 9.73 MB/sec execute 543 sec latency 38.715 ms 2024-09-17T09:48:08.948 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:08 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:08.949 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:08 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:48:08.949 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:08 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:08.949 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:08 smithi078 ceph-mon[63787]: pgmap v259: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.5 MiB/s wr, 463 op/s; 431 KiB/s, 8 objects/s recovering 2024-09-17T09:48:08.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:08 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:08.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:08 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:48:08.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:08 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:08.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:08 smithi136 ceph-mon[49560]: pgmap v259: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.5 MiB/s wr, 463 op/s; 431 KiB/s, 8 objects/s recovering 2024-09-17T09:48:09.851 INFO:teuthology.orchestra.run.smithi078.stdout: 5 270410 9.74 MB/sec execute 544 sec latency 49.657 ms 2024-09-17T09:48:10.851 INFO:teuthology.orchestra.run.smithi078.stdout: 5 270755 9.75 MB/sec execute 545 sec latency 39.324 ms 2024-09-17T09:48:11.561 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-17T09:48:11.759 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:11 smithi078 ceph-mon[63787]: pgmap v260: 97 pgs: 97 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.9 MiB/s wr, 504 op/s; 341 KiB/s, 4 objects/s recovering 2024-09-17T09:48:11.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 271075 9.75 MB/sec execute 546 sec latency 42.547 ms 2024-09-17T09:48:11.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:11 smithi136 ceph-mon[49560]: pgmap v260: 97 pgs: 97 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.9 MiB/s wr, 504 op/s; 341 KiB/s, 4 objects/s recovering 2024-09-17T09:48:12.509 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:48:12.509 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (8m) 106s ago 17m 17.8M - 0.27.0 11f11916f8cd 97db2373ce92 2024-09-17T09:48:12.509 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (5m) 106s ago 17m 6899k - 19.3.0-4963-ge646f729 f271290a0950 da528c5ca445 2024-09-17T09:48:12.509 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi136 smithi136 running (5m) 30s ago 17m 6899k - 19.3.0-4963-ge646f729 f271290a0950 909f405348cc 2024-09-17T09:48:12.509 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (8m) 106s ago 17m 53.0M - dad864ee21e9 db74b0f8e0a7 2024-09-17T09:48:12.510 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.jdrsup smithi078 running (11m) 106s ago 11m 38.4M - 17.2.0 e1d6a67b021e 8f834e15ba88 2024-09-17T09:48:12.510 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi136.smslff smithi136 running (11m) 30s ago 11m 16.7M - 17.2.0 e1d6a67b021e 0c20e2b328d3 2024-09-17T09:48:12.510 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rsqofi smithi078 *:8443,9283,8765 running (7m) 106s ago 18m 519M - 19.3.0-4963-ge646f729 f271290a0950 7dda3108c9a0 2024-09-17T09:48:12.510 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi136.hicmlm smithi136 *:8443,9283,8765 running (6m) 30s ago 17m 456M - 19.3.0-4963-ge646f729 f271290a0950 fc518b58e07a 2024-09-17T09:48:12.510 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (6m) 106s ago 19m 53.2M 2048M 19.3.0-4963-ge646f729 f271290a0950 3ead101d3db1 2024-09-17T09:48:12.510 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi136 smithi136 running (6m) 30s ago 16m 44.3M 2048M 19.3.0-4963-ge646f729 f271290a0950 4fce88842b79 2024-09-17T09:48:12.510 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.tcwbvi smithi078 *:2049 running (11m) 106s ago 11m 284M - 4.0 e1d6a67b021e 30c87cb46555 2024-09-17T09:48:12.510 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi136.regjvc smithi136 *:2049 running (11m) 30s ago 11m 90.2M - 4.0 e1d6a67b021e b88e4dc24376 2024-09-17T09:48:12.510 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (8m) 106s ago 17m 11.0M - 1.7.0 72c9c2088986 52dad3adeeaa 2024-09-17T09:48:12.510 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi136 smithi136 *:9100 running (7m) 30s ago 16m 12.1M - 1.7.0 72c9c2088986 f2bc5856e35a 2024-09-17T09:48:12.510 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (4m) 106s ago 16m 351M 1936M 19.3.0-4963-ge646f729 f271290a0950 1ea81cc809b8 2024-09-17T09:48:12.510 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (3m) 106s ago 15m 290M 1936M 19.3.0-4963-ge646f729 f271290a0950 eda32560334b 2024-09-17T09:48:12.511 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (2m) 106s ago 15m 213M 1936M 19.3.0-4963-ge646f729 f271290a0950 83a32ad7c358 2024-09-17T09:48:12.511 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (117s) 106s ago 14m 14.3M 1936M 19.3.0-4963-ge646f729 f271290a0950 e2cd18998d2b 2024-09-17T09:48:12.511 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi136 running (71s) 30s ago 14m 284M 2704M 19.3.0-4963-ge646f729 f271290a0950 e2a18ad94664 2024-09-17T09:48:12.511 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi136 running (34s) 30s ago 13m 14.4M 2704M 19.3.0-4963-ge646f729 f271290a0950 df9b6954ce0f 2024-09-17T09:48:12.511 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi136 running (13m) 30s ago 13m 556M 2704M 17.2.0 e1d6a67b021e 1698e1567c33 2024-09-17T09:48:12.511 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi136 running (13m) 30s ago 13m 559M 2704M 17.2.0 e1d6a67b021e 5d3b08cecd78 2024-09-17T09:48:12.511 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (6m) 106s ago 16m 53.8M - 2.51.0 1d3b7f56885b 6e5e8c721eb0 2024-09-17T09:48:12.784 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:12 smithi078 ceph-mon[63787]: from='client.34404 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:48:12.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 271377 9.74 MB/sec execute 547 sec latency 48.780 ms 2024-09-17T09:48:12.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:12 smithi136 ceph-mon[49560]: from='client.34404 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:48:13.072 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:48:13.072 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-17T09:48:13.072 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:48:13.072 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:48:13.072 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-17T09:48:13.072 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:48:13.073 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:48:13.073 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-17T09:48:13.073 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-17T09:48:13.073 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 6 2024-09-17T09:48:13.073 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:48:13.073 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-17T09:48:13.073 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:48:13.073 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:48:13.073 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-17T09:48:13.073 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-17T09:48:13.073 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 10 2024-09-17T09:48:13.073 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-17T09:48:13.073 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:48:13.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:13 smithi078 ceph-mon[63787]: from='client.34408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:48:13.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:13 smithi078 ceph-mon[63787]: pgmap v261: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 437 op/s 2024-09-17T09:48:13.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:13 smithi078 ceph-mon[63787]: from='client.34412 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:48:13.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:13 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/1043515197' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:48:13.694 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:48:13.694 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407", 2024-09-17T09:48:13.694 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-17T09:48:13.694 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T09:48:13.694 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-17T09:48:13.694 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-09-17T09:48:13.694 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-17T09:48:13.694 INFO:teuthology.orchestra.run.smithi078.stdout: "mon" 2024-09-17T09:48:13.694 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-17T09:48:13.694 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "12/23 daemons upgraded", 2024-09-17T09:48:13.695 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T09:48:13.695 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-17T09:48:13.695 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:48:13.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 271653 9.74 MB/sec execute 548 sec latency 59.251 ms 2024-09-17T09:48:13.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:13 smithi136 ceph-mon[49560]: from='client.34408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:48:13.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:13 smithi136 ceph-mon[49560]: pgmap v261: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 437 op/s 2024-09-17T09:48:13.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:13 smithi136 ceph-mon[49560]: from='client.34412 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:48:13.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:13 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/1043515197' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:48:14.286 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-17T09:48:14.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:14 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/45390229' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:48:14.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 271981 9.74 MB/sec execute 549 sec latency 51.148 ms 2024-09-17T09:48:14.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:14 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/45390229' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:48:15.586 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:15 smithi078 ceph-mon[63787]: from='client.44335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:48:15.587 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:15 smithi078 ceph-mon[63787]: pgmap v262: 97 pgs: 97 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.4 MiB/s wr, 504 op/s 2024-09-17T09:48:15.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 272592 9.74 MB/sec execute 550 sec latency 30.111 ms 2024-09-17T09:48:15.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:15 smithi136 ceph-mon[49560]: from='client.44335 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:48:15.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:15 smithi136 ceph-mon[49560]: pgmap v262: 97 pgs: 97 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.4 MiB/s wr, 504 op/s 2024-09-17T09:48:16.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 273090 9.74 MB/sec execute 551 sec latency 28.964 ms 2024-09-17T09:48:17.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:17 smithi078 ceph-mon[63787]: pgmap v263: 97 pgs: 97 active+clean; 350 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.7 MiB/s wr, 501 op/s 2024-09-17T09:48:17.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:17 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-17T09:48:17.832 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:17 smithi136 ceph-mon[49560]: pgmap v263: 97 pgs: 97 active+clean; 350 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.7 MiB/s wr, 501 op/s 2024-09-17T09:48:17.832 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:17 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-17T09:48:17.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 273489 9.75 MB/sec execute 552 sec latency 49.607 ms 2024-09-17T09:48:18.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:18 smithi078 ceph-mon[63787]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T09:48:18.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:18 smithi078 ceph-mon[63787]: Upgrade: osd.6 is safe to restart 2024-09-17T09:48:18.840 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:18 smithi136 ceph-mon[49560]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T09:48:18.840 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:18 smithi136 ceph-mon[49560]: Upgrade: osd.6 is safe to restart 2024-09-17T09:48:18.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 273938 9.76 MB/sec execute 553 sec latency 43.759 ms 2024-09-17T09:48:19.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:19 smithi078 ceph-mon[63787]: pgmap v264: 97 pgs: 97 active+clean; 348 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 5.1 MiB/s wr, 536 op/s 2024-09-17T09:48:19.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:19 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:19.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:19 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-17T09:48:19.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:19 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:48:19.855 INFO:teuthology.orchestra.run.smithi078.stdout: 5 274277 9.76 MB/sec execute 554 sec latency 36.717 ms 2024-09-17T09:48:19.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:19 smithi136 ceph-mon[49560]: pgmap v264: 97 pgs: 97 active+clean; 348 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 5.1 MiB/s wr, 536 op/s 2024-09-17T09:48:19.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:19 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:19.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:19 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-17T09:48:19.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:19 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:48:20.658 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:20 smithi078 ceph-mon[63787]: Upgrade: Updating osd.6 2024-09-17T09:48:20.658 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:20 smithi078 ceph-mon[63787]: Deploying daemon osd.6 on smithi136 2024-09-17T09:48:20.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:20 smithi136 ceph-mon[49560]: Upgrade: Updating osd.6 2024-09-17T09:48:20.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:20 smithi136 ceph-mon[49560]: Deploying daemon osd.6 on smithi136 2024-09-17T09:48:20.856 INFO:teuthology.orchestra.run.smithi078.stdout: 5 274637 9.76 MB/sec execute 555 sec latency 37.883 ms 2024-09-17T09:48:21.673 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:21 smithi078 ceph-mon[63787]: pgmap v265: 97 pgs: 97 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.4 MiB/s wr, 529 op/s 2024-09-17T09:48:21.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:21 smithi136 ceph-mon[49560]: pgmap v265: 97 pgs: 97 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.4 MiB/s wr, 529 op/s 2024-09-17T09:48:21.856 INFO:teuthology.orchestra.run.smithi078.stdout: 5 274964 9.76 MB/sec execute 556 sec latency 38.563 ms 2024-09-17T09:48:22.856 INFO:teuthology.orchestra.run.smithi078.stdout: 5 275221 9.76 MB/sec execute 557 sec latency 44.725 ms 2024-09-17T09:48:23.475 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:23 smithi136 systemd[1]: Stopping Ceph osd.6 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:48:23.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:23 smithi078 ceph-mon[63787]: pgmap v266: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.4 MiB/s wr, 524 op/s 2024-09-17T09:48:23.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:23 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:48:23.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:23 smithi078 ceph-mon[63787]: osd.6 marked itself down and dead 2024-09-17T09:48:23.856 INFO:teuthology.orchestra.run.smithi078.stdout: 5 275373 9.74 MB/sec execute 558 sec latency 289.203 ms 2024-09-17T09:48:23.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:23 smithi136 ceph-mon[49560]: pgmap v266: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.4 MiB/s wr, 524 op/s 2024-09-17T09:48:23.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:23 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:48:23.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:23 smithi136 ceph-mon[49560]: osd.6 marked itself down and dead 2024-09-17T09:48:23.976 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:23 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[36113]: 2024-09-17T09:48:23.489+0000 7f465e232700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:48:23.976 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:23 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[36113]: 2024-09-17T09:48:23.489+0000 7f465e232700 -1 osd.6 141 *** Got signal Terminated *** 2024-09-17T09:48:23.976 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:23 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[36113]: 2024-09-17T09:48:23.489+0000 7f465e232700 -1 osd.6 141 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:48:24.590 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:24 smithi136 podman[57019]: 2024-09-17 09:48:24.226119515 +0000 UTC m=+0.857926371 container died 1698e1567c33be692c0a1a22a029b7e782f3a73ad52a85a737d2cccc74af2f5b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, version=8, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, release=754, GIT_CLEAN=True, architecture=x86_64, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, vcs-type=git) 2024-09-17T09:48:24.839 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:24 smithi078 ceph-mon[63787]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:48:24.839 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:24 smithi078 ceph-mon[63787]: osdmap e142: 8 total, 7 up, 8 in 2024-09-17T09:48:24.840 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:24 smithi078 ceph-mon[63787]: pgmap v268: 97 pgs: 1 peering, 9 stale+active+clean, 87 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.6 MiB/s wr, 550 op/s 2024-09-17T09:48:24.840 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:24 smithi078 ceph-mon[63787]: osdmap e143: 8 total, 7 up, 8 in 2024-09-17T09:48:24.856 INFO:teuthology.orchestra.run.smithi078.stdout: 5 275426 9.73 MB/sec execute 559 sec latency 1066.815 ms 2024-09-17T09:48:24.900 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:24 smithi136 ceph-mon[49560]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:48:24.900 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:24 smithi136 ceph-mon[49560]: osdmap e142: 8 total, 7 up, 8 in 2024-09-17T09:48:24.900 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:24 smithi136 ceph-mon[49560]: pgmap v268: 97 pgs: 1 peering, 9 stale+active+clean, 87 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.6 MiB/s wr, 550 op/s 2024-09-17T09:48:24.900 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:24 smithi136 ceph-mon[49560]: osdmap e143: 8 total, 7 up, 8 in 2024-09-17T09:48:25.226 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:24 smithi136 podman[57019]: 2024-09-17 09:48:24.998026972 +0000 UTC m=+1.629833827 container cleanup 1698e1567c33be692c0a1a22a029b7e782f3a73ad52a85a737d2cccc74af2f5b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., GIT_CLEAN=True, name=centos-stream, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, RELEASE=HEAD, release=754, distribution-scope=public) 2024-09-17T09:48:25.226 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:24 smithi136 bash[57019]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6 2024-09-17T09:48:25.627 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:25 smithi136 podman[57049]: 2024-09-17 09:48:25.291319052 +0000 UTC m=+1.067881004 container remove 1698e1567c33be692c0a1a22a029b7e782f3a73ad52a85a737d2cccc74af2f5b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6, name=centos-stream, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, distribution-scope=public, RELEASE=HEAD, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container) 2024-09-17T09:48:25.863 INFO:teuthology.orchestra.run.smithi078.stdout: 5 275753 9.72 MB/sec execute 560 sec latency 59.365 ms 2024-09-17T09:48:25.975 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:25 smithi136 podman[57139]: 2024-09-17 09:48:25.529523922 +0000 UTC m=+0.015903507 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:48:25.976 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:25 smithi136 podman[57139]: 2024-09-17 09:48:25.858994388 +0000 UTC m=+0.345373958 container create a124690013de4dcaf97d31b5d277d367bc36213a079667e4c41b151ff87f78ae (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:48:26.320 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:26 smithi136 podman[57139]: 2024-09-17 09:48:26.112035623 +0000 UTC m=+0.598415203 container init a124690013de4dcaf97d31b5d277d367bc36213a079667e4c41b151ff87f78ae (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-17T09:48:26.321 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:26 smithi136 podman[57139]: 2024-09-17 09:48:26.115829482 +0000 UTC m=+0.602209068 container start a124690013de4dcaf97d31b5d277d367bc36213a079667e4c41b151ff87f78ae (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=) 2024-09-17T09:48:26.321 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:26 smithi136 podman[57139]: 2024-09-17 09:48:26.212114354 +0000 UTC m=+0.698493939 container attach a124690013de4dcaf97d31b5d277d367bc36213a079667e4c41b151ff87f78ae (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-17T09:48:26.597 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:26 smithi136 podman[57139]: 2024-09-17 09:48:26.319234496 +0000 UTC m=+0.805614083 container died a124690013de4dcaf97d31b5d277d367bc36213a079667e4c41b151ff87f78ae (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True) 2024-09-17T09:48:26.636 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:26 smithi078 ceph-mon[63787]: pgmap v270: 97 pgs: 7 active+undersized, 1 peering, 3 stale+active+clean, 11 active+undersized+degraded, 75 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.3 MiB/s wr, 497 op/s; 81/1656 objects degraded (4.891%) 2024-09-17T09:48:26.636 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:26 smithi078 ceph-mon[63787]: Health check failed: Degraded data redundancy: 81/1656 objects degraded (4.891%), 11 pgs degraded (PG_DEGRADED) 2024-09-17T09:48:26.863 INFO:teuthology.orchestra.run.smithi078.stdout: 5 276148 9.72 MB/sec execute 561 sec latency 43.669 ms 2024-09-17T09:48:26.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:26 smithi136 ceph-mon[49560]: pgmap v270: 97 pgs: 7 active+undersized, 1 peering, 3 stale+active+clean, 11 active+undersized+degraded, 75 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.3 MiB/s wr, 497 op/s; 81/1656 objects degraded (4.891%) 2024-09-17T09:48:26.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:26 smithi136 ceph-mon[49560]: Health check failed: Degraded data redundancy: 81/1656 objects degraded (4.891%), 11 pgs degraded (PG_DEGRADED) 2024-09-17T09:48:27.444 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:26 smithi136 podman[57162]: 2024-09-17 09:48:26.983289281 +0000 UTC m=+0.658158217 container remove a124690013de4dcaf97d31b5d277d367bc36213a079667e4c41b151ff87f78ae (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-09-17T09:48:27.444 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:26 smithi136 systemd[1]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.6.service: Deactivated successfully. 2024-09-17T09:48:27.445 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:26 smithi136 systemd[1]: Stopped Ceph osd.6 for 35c0cd22-74d7-11ef-bceb-c7b262605968. 2024-09-17T09:48:27.445 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:26 smithi136 systemd[1]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.6.service: Consumed 5min 18.328s CPU time. 2024-09-17T09:48:27.445 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:27 smithi136 systemd[1]: Starting Ceph osd.6 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:48:27.725 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:27 smithi136 podman[57261]: 2024-09-17 09:48:27.345948912 +0000 UTC m=+0.016587884 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:48:27.726 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:27 smithi136 podman[57261]: 2024-09-17 09:48:27.570376459 +0000 UTC m=+0.241015423 container create a543dcdb867fcfc787eb451fb0b3e44c0dd2ad18eb62f5d4d64144489957e1f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T09:48:27.863 INFO:teuthology.orchestra.run.smithi078.stdout: 5 276450 9.71 MB/sec execute 562 sec latency 61.066 ms 2024-09-17T09:48:28.072 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:27 smithi136 podman[57261]: 2024-09-17 09:48:27.917684151 +0000 UTC m=+0.588323127 container init a543dcdb867fcfc787eb451fb0b3e44c0dd2ad18eb62f5d4d64144489957e1f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6-activate, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T09:48:28.073 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:27 smithi136 podman[57261]: 2024-09-17 09:48:27.922000792 +0000 UTC m=+0.592639758 container start a543dcdb867fcfc787eb451fb0b3e44c0dd2ad18eb62f5d4d64144489957e1f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS) 2024-09-17T09:48:28.073 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:28 smithi136 podman[57261]: 2024-09-17 09:48:28.01373746 +0000 UTC m=+0.684376432 container attach a543dcdb867fcfc787eb451fb0b3e44c0dd2ad18eb62f5d4d64144489957e1f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-17T09:48:28.475 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:28 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6-activate[57276]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:48:28.475 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:28 smithi136 bash[57261]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:48:28.476 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:28 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6-activate[57276]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:48:28.476 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:28 smithi136 bash[57261]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:48:28.863 INFO:teuthology.orchestra.run.smithi078.stdout: 5 276829 9.72 MB/sec execute 563 sec latency 69.075 ms 2024-09-17T09:48:29.157 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:28 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6-activate[57276]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-17T09:48:29.157 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:28 smithi136 bash[57261]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-17T09:48:29.157 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:28 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6-activate[57276]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-17T09:48:29.157 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:28 smithi136 bash[57261]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-17T09:48:29.476 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:29 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6-activate[57276]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-17T09:48:29.476 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:29 smithi136 bash[57261]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-17T09:48:29.476 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:29 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6-activate[57276]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-17T09:48:29.476 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:29 smithi136 bash[57261]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-17T09:48:29.476 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:29 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6-activate[57276]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-17T09:48:29.476 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:29 smithi136 bash[57261]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-17T09:48:29.476 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:29 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6-activate[57276]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-17T09:48:29.476 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:29 smithi136 bash[57261]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-17T09:48:29.476 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:29 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6-activate[57276]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-17T09:48:29.476 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:29 smithi136 bash[57261]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-17T09:48:29.477 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:29 smithi136 podman[57261]: 2024-09-17 09:48:29.188832047 +0000 UTC m=+1.859471014 container died a543dcdb867fcfc787eb451fb0b3e44c0dd2ad18eb62f5d4d64144489957e1f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True) 2024-09-17T09:48:29.722 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:29 smithi078 ceph-mon[63787]: pgmap v271: 97 pgs: 10 active+undersized, 1 peering, 1 stale+active+clean, 19 active+undersized+degraded, 66 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 5.1 MiB/s wr, 565 op/s; 147/1599 objects degraded (9.193%) 2024-09-17T09:48:29.818 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:29 smithi136 ceph-mon[49560]: pgmap v271: 97 pgs: 10 active+undersized, 1 peering, 1 stale+active+clean, 19 active+undersized+degraded, 66 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 5.1 MiB/s wr, 565 op/s; 147/1599 objects degraded (9.193%) 2024-09-17T09:48:29.863 INFO:teuthology.orchestra.run.smithi078.stdout: 5 277241 9.72 MB/sec execute 564 sec latency 73.313 ms 2024-09-17T09:48:30.072 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:29 smithi136 podman[57401]: 2024-09-17 09:48:29.81734756 +0000 UTC m=+0.620843794 container remove a543dcdb867fcfc787eb451fb0b3e44c0dd2ad18eb62f5d4d64144489957e1f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-17T09:48:30.072 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:30 smithi136 podman[57457]: 2024-09-17 09:48:29.937995958 +0000 UTC m=+0.018417072 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:48:30.333 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:30 smithi136 podman[57457]: 2024-09-17 09:48:30.071304873 +0000 UTC m=+0.151725968 container create c477e7043301df66181a9bda379471b3c3b49a4e4890512fe62633f6595891ad (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD) 2024-09-17T09:48:30.333 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:30 smithi136 podman[57457]: 2024-09-17 09:48:30.332534812 +0000 UTC m=+0.412955921 container init c477e7043301df66181a9bda379471b3c3b49a4e4890512fe62633f6595891ad (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72) 2024-09-17T09:48:30.725 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:30 smithi136 podman[57457]: 2024-09-17 09:48:30.336866639 +0000 UTC m=+0.417287744 container start c477e7043301df66181a9bda379471b3c3b49a4e4890512fe62633f6595891ad (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6, org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-17T09:48:30.726 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:30 smithi136 ceph-osd[57475]: -- 172.21.15.136:0/2628273339 <== mon.1 v2:172.21.15.136:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5619de7041e0 con 0x5619de690000 2024-09-17T09:48:30.726 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:30 smithi136 bash[57457]: c477e7043301df66181a9bda379471b3c3b49a4e4890512fe62633f6595891ad 2024-09-17T09:48:30.726 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:30 smithi136 systemd[1]: Started Ceph osd.6 for 35c0cd22-74d7-11ef-bceb-c7b262605968. 2024-09-17T09:48:30.863 INFO:teuthology.orchestra.run.smithi078.stdout: 5 277666 9.73 MB/sec execute 565 sec latency 38.441 ms 2024-09-17T09:48:31.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:31 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:31.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:31 smithi136 ceph-mon[49560]: pgmap v272: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.5 MiB/s wr, 509 op/s; 187/1560 objects degraded (11.987%) 2024-09-17T09:48:31.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:31 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:31.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:31 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:48:31.746 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:31 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:31.746 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:31 smithi078 ceph-mon[63787]: pgmap v272: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.5 MiB/s wr, 509 op/s; 187/1560 objects degraded (11.987%) 2024-09-17T09:48:31.747 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:31 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:31.747 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:31 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:48:31.863 INFO:teuthology.orchestra.run.smithi078.stdout: 5 278021 9.73 MB/sec execute 566 sec latency 41.307 ms 2024-09-17T09:48:32.302 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:32 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:48:32.212+0000 7f07a88bc640 -1 Falling back to public interface 2024-09-17T09:48:32.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:32 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 187/1560 objects degraded (11.987%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T09:48:32.788 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:32 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 187/1560 objects degraded (11.987%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T09:48:32.863 INFO:teuthology.orchestra.run.smithi078.stdout: 5 278393 9.73 MB/sec execute 567 sec latency 37.756 ms 2024-09-17T09:48:33.551 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:33 smithi078 ceph-mon[63787]: pgmap v273: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 3.2 MiB/s wr, 280 op/s; 193/1572 objects degraded (12.277%) 2024-09-17T09:48:33.863 INFO:teuthology.orchestra.run.smithi078.stdout: 5 278728 9.73 MB/sec execute 568 sec latency 46.384 ms 2024-09-17T09:48:33.930 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:33 smithi136 ceph-mon[49560]: pgmap v273: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 3.2 MiB/s wr, 280 op/s; 193/1572 objects degraded (12.277%) 2024-09-17T09:48:34.863 INFO:teuthology.orchestra.run.smithi078.stdout: 5 279021 9.73 MB/sec execute 569 sec latency 47.794 ms 2024-09-17T09:48:35.343 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:35 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:35.343 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:35 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:35.344 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:35 smithi078 ceph-mon[63787]: pgmap v274: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 4.9 MiB/s wr, 517 op/s; 211/1683 objects degraded (12.537%) 2024-09-17T09:48:35.344 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:35 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:35.344 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:35 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:35.645 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:35 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:35.645 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:35 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:35.645 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:35 smithi136 ceph-mon[49560]: pgmap v274: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 4.9 MiB/s wr, 517 op/s; 211/1683 objects degraded (12.537%) 2024-09-17T09:48:35.645 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:35 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:35.645 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:35 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:35.863 INFO:teuthology.orchestra.run.smithi078.stdout: 5 279324 9.72 MB/sec execute 570 sec latency 68.352 ms 2024-09-17T09:48:36.864 INFO:teuthology.orchestra.run.smithi078.stdout: 5 279711 9.72 MB/sec execute 571 sec latency 44.888 ms 2024-09-17T09:48:37.654 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:37 smithi078 ceph-mon[63787]: pgmap v275: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 453 op/s; 207/1677 objects degraded (12.343%) 2024-09-17T09:48:37.655 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:37 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 207/1677 objects degraded (12.343%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T09:48:37.864 INFO:teuthology.orchestra.run.smithi078.stdout: 5 280084 9.71 MB/sec execute 572 sec latency 53.686 ms 2024-09-17T09:48:37.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:37 smithi136 ceph-mon[49560]: pgmap v275: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 453 op/s; 207/1677 objects degraded (12.343%) 2024-09-17T09:48:37.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:37 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 207/1677 objects degraded (12.343%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T09:48:38.864 INFO:teuthology.orchestra.run.smithi078.stdout: 5 280586 9.72 MB/sec execute 573 sec latency 64.196 ms 2024-09-17T09:48:38.922 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:38.922 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:48:38.922 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:38.922 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:38 smithi078 ceph-mon[63787]: pgmap v276: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.7 MiB/s wr, 461 op/s; 197/1641 objects degraded (12.005%) 2024-09-17T09:48:38.922 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:38.922 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:48:38.922 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:48:38.922 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:38.922 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:38.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:38.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:48:38.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:38.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:38 smithi136 ceph-mon[49560]: pgmap v276: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.7 MiB/s wr, 461 op/s; 197/1641 objects degraded (12.005%) 2024-09-17T09:48:38.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:38.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:48:38.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:48:38.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:38.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:39.864 INFO:teuthology.orchestra.run.smithi078.stdout: 5 281057 9.74 MB/sec execute 574 sec latency 45.266 ms 2024-09-17T09:48:39.892 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:39 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:48:39.893 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:39 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:48:39.893 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:39 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:48:39.893 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:39 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:48:39.893 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:39 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-17T09:48:39.893 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:39 smithi078 ceph-mon[63787]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T09:48:39.893 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:39 smithi078 ceph-mon[63787]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-17T09:48:39.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:39 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:48:39.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:39 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:48:39.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:39 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:48:39.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:39 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:48:39.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:39 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-17T09:48:39.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:39 smithi136 ceph-mon[49560]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T09:48:39.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:39 smithi136 ceph-mon[49560]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-17T09:48:40.864 INFO:teuthology.orchestra.run.smithi078.stdout: 5 281388 9.74 MB/sec execute 575 sec latency 35.059 ms 2024-09-17T09:48:40.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:40 smithi078 ceph-mon[63787]: pgmap v277: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.9 MiB/s wr, 455 op/s; 194/1614 objects degraded (12.020%) 2024-09-17T09:48:40.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:40 smithi136 ceph-mon[49560]: pgmap v277: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.9 MiB/s wr, 455 op/s; 194/1614 objects degraded (12.020%) 2024-09-17T09:48:40.976 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:40 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:48:40.510+0000 7f07a88bc640 -1 osd.6 0 read_superblock omap replica is missing. 2024-09-17T09:48:41.864 INFO:teuthology.orchestra.run.smithi078.stdout: 5 281724 9.74 MB/sec execute 576 sec latency 46.267 ms 2024-09-17T09:48:42.171 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:41 smithi078 ceph-mon[63787]: from='osd.6 [v2:172.21.15.136:6816/3860383094,v1:172.21.15.136:6817/3860383094]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-17T09:48:42.171 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:41 smithi078 ceph-mon[63787]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-17T09:48:42.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:41 smithi136 ceph-mon[49560]: from='osd.6 [v2:172.21.15.136:6816/3860383094,v1:172.21.15.136:6817/3860383094]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-17T09:48:42.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:41 smithi136 ceph-mon[49560]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-17T09:48:42.226 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:41 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:48:41.874+0000 7f07a88bc640 -1 osd.6 141 log_to_monitors true 2024-09-17T09:48:42.725 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:48:42 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:48:42.262+0000 7f079c65e640 -1 osd.6 141 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T09:48:42.864 INFO:teuthology.orchestra.run.smithi078.stdout: 5 281968 9.73 MB/sec execute 577 sec latency 103.362 ms 2024-09-17T09:48:43.189 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:42 smithi078 ceph-mon[63787]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-17T09:48:43.190 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:42 smithi078 ceph-mon[63787]: osdmap e144: 8 total, 7 up, 8 in 2024-09-17T09:48:43.190 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:42 smithi078 ceph-mon[63787]: from='osd.6 [v2:172.21.15.136:6816/3860383094,v1:172.21.15.136:6817/3860383094]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi136", "root=default"]} : dispatch 2024-09-17T09:48:43.190 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:42 smithi078 ceph-mon[63787]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi136", "root=default"]} : dispatch 2024-09-17T09:48:43.190 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:42 smithi078 ceph-mon[63787]: from='osd.6 ' entity='osd.6' 2024-09-17T09:48:43.190 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:42 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 194/1614 objects degraded (12.020%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T09:48:43.190 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:42 smithi078 ceph-mon[63787]: pgmap v279: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 344 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.2 MiB/s wr, 704 op/s; 194/1629 objects degraded (11.909%) 2024-09-17T09:48:43.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:42 smithi136 ceph-mon[49560]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-17T09:48:43.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:42 smithi136 ceph-mon[49560]: osdmap e144: 8 total, 7 up, 8 in 2024-09-17T09:48:43.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:42 smithi136 ceph-mon[49560]: from='osd.6 [v2:172.21.15.136:6816/3860383094,v1:172.21.15.136:6817/3860383094]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi136", "root=default"]} : dispatch 2024-09-17T09:48:43.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:42 smithi136 ceph-mon[49560]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi136", "root=default"]} : dispatch 2024-09-17T09:48:43.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:42 smithi136 ceph-mon[49560]: from='osd.6 ' entity='osd.6' 2024-09-17T09:48:43.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:42 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 194/1614 objects degraded (12.020%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T09:48:43.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:42 smithi136 ceph-mon[49560]: pgmap v279: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 344 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.2 MiB/s wr, 704 op/s; 194/1629 objects degraded (11.909%) 2024-09-17T09:48:43.864 INFO:teuthology.orchestra.run.smithi078.stdout: 5 282203 9.72 MB/sec execute 578 sec latency 565.718 ms 2024-09-17T09:48:43.957 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:43 smithi078 ceph-mon[63787]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:48:43.958 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:43 smithi078 ceph-mon[63787]: osd.6 [v2:172.21.15.136:6816/3860383094,v1:172.21.15.136:6817/3860383094] boot 2024-09-17T09:48:43.958 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:43 smithi078 ceph-mon[63787]: osdmap e145: 8 total, 8 up, 8 in 2024-09-17T09:48:43.958 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:43 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-17T09:48:44.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:43 smithi136 ceph-mon[49560]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:48:44.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:43 smithi136 ceph-mon[49560]: osd.6 [v2:172.21.15.136:6816/3860383094,v1:172.21.15.136:6817/3860383094] boot 2024-09-17T09:48:44.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:43 smithi136 ceph-mon[49560]: osdmap e145: 8 total, 8 up, 8 in 2024-09-17T09:48:44.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:43 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-17T09:48:44.779 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-17T09:48:44.864 INFO:teuthology.orchestra.run.smithi078.stdout: 5 282331 9.71 MB/sec execute 579 sec latency 1551.720 ms 2024-09-17T09:48:45.538 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:45 smithi078 ceph-mon[63787]: osdmap e146: 8 total, 8 up, 8 in 2024-09-17T09:48:45.539 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:45 smithi078 ceph-mon[63787]: pgmap v282: 97 pgs: 4 peering, 10 active+undersized, 21 active+undersized+degraded, 62 active+clean; 348 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 6.6 MiB/s wr, 695 op/s; 165/1674 objects degraded (9.857%) 2024-09-17T09:48:45.597 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:48:45.597 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (9m) 2m ago 18m 17.8M - 0.27.0 11f11916f8cd 97db2373ce92 2024-09-17T09:48:45.597 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (5m) 2m ago 18m 6899k - 19.3.0-4963-ge646f729 f271290a0950 da528c5ca445 2024-09-17T09:48:45.597 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi136 smithi136 running (5m) 11s ago 17m 6899k - 19.3.0-4963-ge646f729 f271290a0950 909f405348cc 2024-09-17T09:48:45.597 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (9m) 2m ago 18m 53.0M - dad864ee21e9 db74b0f8e0a7 2024-09-17T09:48:45.597 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.jdrsup smithi078 running (12m) 2m ago 12m 38.4M - 17.2.0 e1d6a67b021e 8f834e15ba88 2024-09-17T09:48:45.597 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi136.smslff smithi136 running (12m) 11s ago 12m 16.9M - 17.2.0 e1d6a67b021e 0c20e2b328d3 2024-09-17T09:48:45.597 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rsqofi smithi078 *:8443,9283,8765 running (8m) 2m ago 19m 519M - 19.3.0-4963-ge646f729 f271290a0950 7dda3108c9a0 2024-09-17T09:48:45.597 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi136.hicmlm smithi136 *:8443,9283,8765 running (7m) 11s ago 17m 456M - 19.3.0-4963-ge646f729 f271290a0950 fc518b58e07a 2024-09-17T09:48:45.597 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (7m) 2m ago 19m 53.2M 2048M 19.3.0-4963-ge646f729 f271290a0950 3ead101d3db1 2024-09-17T09:48:45.597 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi136 smithi136 running (6m) 11s ago 17m 45.1M 2048M 19.3.0-4963-ge646f729 f271290a0950 4fce88842b79 2024-09-17T09:48:45.597 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.tcwbvi smithi078 *:2049 running (11m) 2m ago 11m 284M - 4.0 e1d6a67b021e 30c87cb46555 2024-09-17T09:48:45.598 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi136.regjvc smithi136 *:2049 running (11m) 11s ago 11m 90.3M - 4.0 e1d6a67b021e b88e4dc24376 2024-09-17T09:48:45.598 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (8m) 2m ago 17m 11.0M - 1.7.0 72c9c2088986 52dad3adeeaa 2024-09-17T09:48:45.598 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi136 smithi136 *:9100 running (8m) 11s ago 17m 11.1M - 1.7.0 72c9c2088986 f2bc5856e35a 2024-09-17T09:48:45.598 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (4m) 2m ago 16m 351M 1936M 19.3.0-4963-ge646f729 f271290a0950 1ea81cc809b8 2024-09-17T09:48:45.598 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (4m) 2m ago 16m 290M 1936M 19.3.0-4963-ge646f729 f271290a0950 eda32560334b 2024-09-17T09:48:45.598 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (3m) 2m ago 15m 213M 1936M 19.3.0-4963-ge646f729 f271290a0950 83a32ad7c358 2024-09-17T09:48:45.598 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (2m) 2m ago 15m 14.3M 1936M 19.3.0-4963-ge646f729 f271290a0950 e2cd18998d2b 2024-09-17T09:48:45.598 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi136 running (105s) 11s ago 14m 313M 2704M 19.3.0-4963-ge646f729 f271290a0950 e2a18ad94664 2024-09-17T09:48:45.598 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi136 running (67s) 11s ago 14m 285M 2704M 19.3.0-4963-ge646f729 f271290a0950 df9b6954ce0f 2024-09-17T09:48:45.598 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi136 running (15s) 11s ago 14m 11.5M 2704M 19.3.0-4963-ge646f729 f271290a0950 c477e7043301 2024-09-17T09:48:45.598 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi136 running (13m) 11s ago 13m 632M 2704M 17.2.0 e1d6a67b021e 5d3b08cecd78 2024-09-17T09:48:45.599 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (7m) 2m ago 17m 53.8M - 2.51.0 1d3b7f56885b 6e5e8c721eb0 2024-09-17T09:48:45.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:45 smithi136 ceph-mon[49560]: osdmap e146: 8 total, 8 up, 8 in 2024-09-17T09:48:45.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:45 smithi136 ceph-mon[49560]: pgmap v282: 97 pgs: 4 peering, 10 active+undersized, 21 active+undersized+degraded, 62 active+clean; 348 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 6.6 MiB/s wr, 695 op/s; 165/1674 objects degraded (9.857%) 2024-09-17T09:48:45.864 INFO:teuthology.orchestra.run.smithi078.stdout: 5 282411 9.69 MB/sec execute 580 sec latency 2105.114 ms 2024-09-17T09:48:46.246 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:48:46.246 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-17T09:48:46.246 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:48:46.246 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:48:46.246 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-17T09:48:46.246 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:48:46.246 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:48:46.246 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-17T09:48:46.246 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T09:48:46.246 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 7 2024-09-17T09:48:46.246 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:48:46.247 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-17T09:48:46.247 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:48:46.247 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:48:46.247 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-17T09:48:46.247 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-17T09:48:46.247 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 11 2024-09-17T09:48:46.247 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-17T09:48:46.247 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:48:46.304 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:46 smithi078 ceph-mon[63787]: from='client.34430 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:48:46.305 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:46 smithi078 ceph-mon[63787]: from='client.34434 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:48:46.305 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:46 smithi078 ceph-mon[63787]: osdmap e147: 8 total, 8 up, 8 in 2024-09-17T09:48:46.305 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:46 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/2008870285' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:48:46.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:46 smithi136 ceph-mon[49560]: from='client.34430 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:48:46.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:46 smithi136 ceph-mon[49560]: from='client.34434 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:48:46.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:46 smithi136 ceph-mon[49560]: osdmap e147: 8 total, 8 up, 8 in 2024-09-17T09:48:46.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:46 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/2008870285' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:48:46.753 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:48:46.753 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407", 2024-09-17T09:48:46.754 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-17T09:48:46.754 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T09:48:46.754 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-17T09:48:46.754 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-09-17T09:48:46.754 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-17T09:48:46.754 INFO:teuthology.orchestra.run.smithi078.stdout: "mon" 2024-09-17T09:48:46.754 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-17T09:48:46.754 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "13/23 daemons upgraded", 2024-09-17T09:48:46.754 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T09:48:46.754 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-17T09:48:46.754 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:48:46.864 INFO:teuthology.orchestra.run.smithi078.stdout: 5 282607 9.69 MB/sec execute 581 sec latency 559.004 ms 2024-09-17T09:48:47.228 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_WARN Degraded data redundancy: 165/1674 objects degraded (9.857%), 21 pgs degraded 2024-09-17T09:48:47.228 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 165/1674 objects degraded (9.857%), 21 pgs degraded 2024-09-17T09:48:47.228 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.2 is active+undersized+degraded, acting [5,1] 2024-09-17T09:48:47.228 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.6 is active+undersized+degraded, acting [1,4] 2024-09-17T09:48:47.228 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.7 is active+undersized+degraded, acting [7,2] 2024-09-17T09:48:47.229 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.11 is active+undersized+degraded, acting [4,1] 2024-09-17T09:48:47.229 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.14 is active+undersized+degraded, acting [3,5] 2024-09-17T09:48:47.229 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.16 is active+undersized+degraded, acting [5,2] 2024-09-17T09:48:47.229 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.17 is active+undersized+degraded, acting [5,2] 2024-09-17T09:48:47.229 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.19 is active+undersized+degraded, acting [3,0] 2024-09-17T09:48:47.229 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1a is active+undersized+degraded, acting [4,7] 2024-09-17T09:48:47.229 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1b is active+undersized+degraded, acting [3,7] 2024-09-17T09:48:47.229 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.0 is active+undersized+degraded, acting [1,2] 2024-09-17T09:48:47.229 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.2 is active+undersized+degraded, acting [3,5] 2024-09-17T09:48:47.229 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.3 is active+undersized+degraded, acting [4,0] 2024-09-17T09:48:47.229 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.a is active+undersized+degraded, acting [4,1] 2024-09-17T09:48:47.229 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.c is active+undersized+degraded, acting [5,3] 2024-09-17T09:48:47.229 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.10 is active+undersized+degraded, acting [0,5] 2024-09-17T09:48:47.230 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.14 is active+undersized+degraded, acting [4,7] 2024-09-17T09:48:47.230 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1d is active+undersized+degraded, acting [5,4] 2024-09-17T09:48:47.230 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1e is active+undersized+degraded, acting [3,2] 2024-09-17T09:48:47.230 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.5 is active+undersized+degraded, acting [4,7] 2024-09-17T09:48:47.230 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.a is active+undersized+degraded, acting [1,7] 2024-09-17T09:48:47.478 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:47 smithi078 ceph-mon[63787]: from='client.34438 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:48:47.478 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:47 smithi078 ceph-mon[63787]: osdmap e148: 8 total, 8 up, 8 in 2024-09-17T09:48:47.479 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:47 smithi078 ceph-mon[63787]: pgmap v285: 97 pgs: 2 active+undersized+remapped, 4 peering, 4 active+undersized, 15 active+undersized+degraded, 72 active+clean; 348 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 510 B/s rd, 3.5 MiB/s wr, 142 op/s; 109/1632 objects degraded (6.679%); 348 KiB/s, 21 objects/s recovering 2024-09-17T09:48:47.479 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:47 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/522856684' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:48:47.627 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:47 smithi136 ceph-mon[49560]: from='client.34438 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:48:47.627 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:47 smithi136 ceph-mon[49560]: osdmap e148: 8 total, 8 up, 8 in 2024-09-17T09:48:47.627 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:47 smithi136 ceph-mon[49560]: pgmap v285: 97 pgs: 2 active+undersized+remapped, 4 peering, 4 active+undersized, 15 active+undersized+degraded, 72 active+clean; 348 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 510 B/s rd, 3.5 MiB/s wr, 142 op/s; 109/1632 objects degraded (6.679%); 348 KiB/s, 21 objects/s recovering 2024-09-17T09:48:47.627 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:47 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/522856684' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:48:47.864 INFO:teuthology.orchestra.run.smithi078.stdout: 5 282818 9.69 MB/sec execute 582 sec latency 1016.834 ms 2024-09-17T09:48:48.514 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:48 smithi078 ceph-mon[63787]: from='client.34446 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:48:48.515 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:48 smithi078 ceph-mon[63787]: osdmap e149: 8 total, 8 up, 8 in 2024-09-17T09:48:48.515 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:48 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 109/1632 objects degraded (6.679%), 15 pgs degraded (PG_DEGRADED) 2024-09-17T09:48:48.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:48 smithi136 ceph-mon[49560]: from='client.34446 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:48:48.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:48 smithi136 ceph-mon[49560]: osdmap e149: 8 total, 8 up, 8 in 2024-09-17T09:48:48.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:48 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 109/1632 objects degraded (6.679%), 15 pgs degraded (PG_DEGRADED) 2024-09-17T09:48:48.864 INFO:teuthology.orchestra.run.smithi078.stdout: 5 283100 9.68 MB/sec execute 583 sec latency 72.634 ms 2024-09-17T09:48:49.522 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:49 smithi078 ceph-mon[63787]: pgmap v287: 97 pgs: 2 active+undersized+remapped, 5 peering, 90 active+clean; 344 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 3.1 MiB/s wr, 332 op/s; 2.9 MiB/s, 63 keys/s, 49 objects/s recovering 2024-09-17T09:48:49.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:49 smithi136 ceph-mon[49560]: pgmap v287: 97 pgs: 2 active+undersized+remapped, 5 peering, 90 active+clean; 344 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 3.1 MiB/s wr, 332 op/s; 2.9 MiB/s, 63 keys/s, 49 objects/s recovering 2024-09-17T09:48:49.864 INFO:teuthology.orchestra.run.smithi078.stdout: 5 283504 9.67 MB/sec execute 584 sec latency 59.588 ms 2024-09-17T09:48:50.560 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:50 smithi078 ceph-mon[63787]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 109/1632 objects degraded (6.679%), 15 pgs degraded) 2024-09-17T09:48:50.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:50 smithi078 ceph-mon[63787]: Cluster is now healthy 2024-09-17T09:48:50.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:50 smithi136 ceph-mon[49560]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 109/1632 objects degraded (6.679%), 15 pgs degraded) 2024-09-17T09:48:50.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:50 smithi136 ceph-mon[49560]: Cluster is now healthy 2024-09-17T09:48:50.864 INFO:teuthology.orchestra.run.smithi078.stdout: 5 283755 9.66 MB/sec execute 585 sec latency 72.673 ms 2024-09-17T09:48:51.321 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:51 smithi078 ceph-mon[63787]: pgmap v288: 97 pgs: 2 active+undersized+remapped, 1 peering, 94 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 2.3 MiB/s wr, 250 op/s; 2.7 MiB/s, 44 keys/s, 42 objects/s recovering 2024-09-17T09:48:51.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:51 smithi136 ceph-mon[49560]: pgmap v288: 97 pgs: 2 active+undersized+remapped, 1 peering, 94 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 2.3 MiB/s wr, 250 op/s; 2.7 MiB/s, 44 keys/s, 42 objects/s recovering 2024-09-17T09:48:51.864 INFO:teuthology.orchestra.run.smithi078.stdout: 5 284086 9.67 MB/sec execute 586 sec latency 38.160 ms 2024-09-17T09:48:52.864 INFO:teuthology.orchestra.run.smithi078.stdout: 5 284421 9.67 MB/sec execute 587 sec latency 50.635 ms 2024-09-17T09:48:53.641 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:53 smithi078 ceph-mon[63787]: pgmap v289: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.3 MiB/s wr, 435 op/s; 2.2 MiB/s, 37 keys/s, 35 objects/s recovering 2024-09-17T09:48:53.641 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:53 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:53.641 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:53 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:48:53.864 INFO:teuthology.orchestra.run.smithi078.stdout: 5 284765 9.68 MB/sec execute 588 sec latency 69.544 ms 2024-09-17T09:48:53.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:53 smithi136 ceph-mon[49560]: pgmap v289: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.3 MiB/s wr, 435 op/s; 2.2 MiB/s, 37 keys/s, 35 objects/s recovering 2024-09-17T09:48:53.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:53 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:53.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:53 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:48:54.660 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:54 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-17T09:48:54.671 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:54 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-17T09:48:54.864 INFO:teuthology.orchestra.run.smithi078.stdout: 5 285010 9.68 MB/sec execute 589 sec latency 72.764 ms 2024-09-17T09:48:55.689 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:55 smithi078 ceph-mon[63787]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T09:48:55.690 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:55 smithi078 ceph-mon[63787]: Upgrade: osd.7 is safe to restart 2024-09-17T09:48:55.690 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:55 smithi078 ceph-mon[63787]: pgmap v290: 97 pgs: 97 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 3.9 MiB/s wr, 427 op/s; 1.8 MiB/s, 32 keys/s, 20 objects/s recovering 2024-09-17T09:48:55.690 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:55 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:55.690 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:55 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-17T09:48:55.690 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:55 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:48:55.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:55 smithi136 ceph-mon[49560]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T09:48:55.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:55 smithi136 ceph-mon[49560]: Upgrade: osd.7 is safe to restart 2024-09-17T09:48:55.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:55 smithi136 ceph-mon[49560]: pgmap v290: 97 pgs: 97 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 3.9 MiB/s wr, 427 op/s; 1.8 MiB/s, 32 keys/s, 20 objects/s recovering 2024-09-17T09:48:55.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:55 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:55.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:55 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-17T09:48:55.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:55 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:48:55.864 INFO:teuthology.orchestra.run.smithi078.stdout: 5 285231 9.67 MB/sec execute 590 sec latency 62.089 ms 2024-09-17T09:48:56.716 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:56 smithi078 ceph-mon[63787]: Upgrade: Updating osd.7 2024-09-17T09:48:56.717 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:56 smithi078 ceph-mon[63787]: Deploying daemon osd.7 on smithi136 2024-09-17T09:48:56.864 INFO:teuthology.orchestra.run.smithi078.stdout: 5 285472 9.66 MB/sec execute 591 sec latency 46.911 ms 2024-09-17T09:48:56.936 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:56 smithi136 ceph-mon[49560]: Upgrade: Updating osd.7 2024-09-17T09:48:56.936 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:56 smithi136 ceph-mon[49560]: Deploying daemon osd.7 on smithi136 2024-09-17T09:48:57.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:57 smithi136 ceph-mon[49560]: pgmap v291: 97 pgs: 97 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.8 MiB/s wr, 374 op/s; 1.5 MiB/s, 26 keys/s, 17 objects/s recovering 2024-09-17T09:48:57.756 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:57 smithi078 ceph-mon[63787]: pgmap v291: 97 pgs: 97 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.8 MiB/s wr, 374 op/s; 1.5 MiB/s, 26 keys/s, 17 objects/s recovering 2024-09-17T09:48:57.864 INFO:teuthology.orchestra.run.smithi078.stdout: 5 285726 9.65 MB/sec execute 592 sec latency 61.735 ms 2024-09-17T09:48:58.864 INFO:teuthology.orchestra.run.smithi078.stdout: 5 285948 9.64 MB/sec execute 593 sec latency 77.201 ms 2024-09-17T09:48:59.022 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:58 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:59.023 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:58 smithi078 ceph-mon[63787]: pgmap v292: 97 pgs: 97 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 4.4 MiB/s wr, 439 op/s; 1.2 MiB/s, 24 keys/s, 11 objects/s recovering 2024-09-17T09:48:59.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:58 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:48:59.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:58 smithi136 ceph-mon[49560]: pgmap v292: 97 pgs: 97 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 4.4 MiB/s wr, 439 op/s; 1.2 MiB/s, 24 keys/s, 11 objects/s recovering 2024-09-17T09:48:59.226 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:48:59 smithi136 systemd[1]: Stopping Ceph osd.7 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:48:59.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:48:59 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[39162]: 2024-09-17T09:48:59.476+0000 7f6a2eeba700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:48:59.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:48:59 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[39162]: 2024-09-17T09:48:59.476+0000 7f6a2eeba700 -1 osd.7 149 *** Got signal Terminated *** 2024-09-17T09:48:59.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:48:59 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[39162]: 2024-09-17T09:48:59.476+0000 7f6a2eeba700 -1 osd.7 149 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:48:59.865 INFO:teuthology.orchestra.run.smithi078.stdout: 5 286216 9.64 MB/sec execute 594 sec latency 68.318 ms 2024-09-17T09:49:00.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:48:59 smithi078 ceph-mon[63787]: osd.7 marked itself down and dead 2024-09-17T09:49:00.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:48:59 smithi136 ceph-mon[49560]: osd.7 marked itself down and dead 2024-09-17T09:49:00.837 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:00 smithi136 podman[59037]: 2024-09-17 09:49:00.555734338 +0000 UTC m=+1.215915967 container died 5d3b08cecd78f483b9ab87dadfa8e6090c999e9e86beabc68819187237a17cc3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7, io.buildah.version=1.19.8, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, version=8, release=754, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , name=centos-stream, distribution-scope=public, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD) 2024-09-17T09:49:00.864 INFO:teuthology.orchestra.run.smithi078.stdout: 5 286216 9.63 MB/sec execute 595 sec latency 1015.382 ms 2024-09-17T09:49:01.109 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:00 smithi078 ceph-mon[63787]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:49:01.109 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:00 smithi078 ceph-mon[63787]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-17T09:49:01.109 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:00 smithi078 ceph-mon[63787]: osdmap e150: 8 total, 7 up, 8 in 2024-09-17T09:49:01.110 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:00 smithi078 ceph-mon[63787]: pgmap v294: 97 pgs: 13 stale+active+clean, 84 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.4 MiB/s wr, 436 op/s 2024-09-17T09:49:01.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:00 smithi136 ceph-mon[49560]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T09:49:01.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:00 smithi136 ceph-mon[49560]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-17T09:49:01.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:00 smithi136 ceph-mon[49560]: osdmap e150: 8 total, 7 up, 8 in 2024-09-17T09:49:01.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:00 smithi136 ceph-mon[49560]: pgmap v294: 97 pgs: 13 stale+active+clean, 84 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.4 MiB/s wr, 436 op/s 2024-09-17T09:49:01.226 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:00 smithi136 podman[59037]: 2024-09-17 09:49:00.944251002 +0000 UTC m=+1.604432626 container cleanup 5d3b08cecd78f483b9ab87dadfa8e6090c999e9e86beabc68819187237a17cc3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, io.openshift.expose-services=, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, ceph=True, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, RELEASE=HEAD, vcs-type=git, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8) 2024-09-17T09:49:01.227 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:00 smithi136 bash[59037]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7 2024-09-17T09:49:01.619 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:01 smithi136 podman[59057]: 2024-09-17 09:49:01.229357214 +0000 UTC m=+0.668114397 container remove 5d3b08cecd78f483b9ab87dadfa8e6090c999e9e86beabc68819187237a17cc3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7, io.buildah.version=1.19.8, io.openshift.expose-services=, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, distribution-scope=public, ceph=True, GIT_CLEAN=True, vcs-type=git, com.redhat.component=centos-stream-container, name=centos-stream, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, release=754, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, io.openshift.tags=base centos centos-stream) 2024-09-17T09:49:01.864 INFO:teuthology.orchestra.run.smithi078.stdout: 5 286561 9.63 MB/sec execute 596 sec latency 1057.568 ms 2024-09-17T09:49:01.872 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:01 smithi078 ceph-mon[63787]: osdmap e151: 8 total, 7 up, 8 in 2024-09-17T09:49:01.965 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:01 smithi136 ceph-mon[49560]: osdmap e151: 8 total, 7 up, 8 in 2024-09-17T09:49:01.965 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:01 smithi136 podman[59139]: 2024-09-17 09:49:01.522769853 +0000 UTC m=+0.020444904 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:49:01.965 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:01 smithi136 podman[59139]: 2024-09-17 09:49:01.647518821 +0000 UTC m=+0.145193865 container create 13e47810247f3a898244e1921600e7df0071e017e78f58238ce8c33b62342e88 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-17T09:49:02.226 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:01 smithi136 podman[59139]: 2024-09-17 09:49:01.963685758 +0000 UTC m=+0.461360813 container init 13e47810247f3a898244e1921600e7df0071e017e78f58238ce8c33b62342e88 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-17T09:49:02.226 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:01 smithi136 podman[59139]: 2024-09-17 09:49:01.968696174 +0000 UTC m=+0.466371218 container start 13e47810247f3a898244e1921600e7df0071e017e78f58238ce8c33b62342e88 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True) 2024-09-17T09:49:02.226 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:02 smithi136 podman[59139]: 2024-09-17 09:49:02.084778288 +0000 UTC m=+0.582453326 container attach 13e47810247f3a898244e1921600e7df0071e017e78f58238ce8c33b62342e88 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-17T09:49:02.226 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:02 smithi136 podman[59139]: 2024-09-17 09:49:02.197293988 +0000 UTC m=+0.694969043 container died 13e47810247f3a898244e1921600e7df0071e017e78f58238ce8c33b62342e88 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T09:49:02.864 INFO:teuthology.orchestra.run.smithi078.stdout: 5 286925 9.62 MB/sec execute 597 sec latency 48.443 ms 2024-09-17T09:49:02.900 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:02 smithi078 ceph-mon[63787]: pgmap v296: 97 pgs: 7 peering, 4 stale+active+clean, 18 active+undersized+degraded, 68 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 3.6 MiB/s wr, 342 op/s; 110/1512 objects degraded (7.275%) 2024-09-17T09:49:03.158 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:02 smithi136 ceph-mon[49560]: pgmap v296: 97 pgs: 7 peering, 4 stale+active+clean, 18 active+undersized+degraded, 68 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 3.6 MiB/s wr, 342 op/s; 110/1512 objects degraded (7.275%) 2024-09-17T09:49:03.158 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:02 smithi136 podman[59162]: 2024-09-17 09:49:02.906626169 +0000 UTC m=+0.703697336 container remove 13e47810247f3a898244e1921600e7df0071e017e78f58238ce8c33b62342e88 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:49:03.158 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:02 smithi136 systemd[1]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.7.service: Deactivated successfully. 2024-09-17T09:49:03.158 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:02 smithi136 systemd[1]: Stopped Ceph osd.7 for 35c0cd22-74d7-11ef-bceb-c7b262605968. 2024-09-17T09:49:03.158 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:02 smithi136 systemd[1]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.7.service: Consumed 7min 31.499s CPU time. 2024-09-17T09:49:03.442 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:03 smithi136 systemd[1]: Starting Ceph osd.7 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:49:03.442 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:03 smithi136 podman[59260]: 2024-09-17 09:49:03.296929567 +0000 UTC m=+0.020259364 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:49:03.713 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:03 smithi136 podman[59260]: 2024-09-17 09:49:03.44162831 +0000 UTC m=+0.164958094 container create e44d804a844772a54858cbf67721f8afe84fbde8a27a09b2ea25a8b8c4e0e188 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7-activate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:49:03.864 INFO:teuthology.orchestra.run.smithi078.stdout: 5 287222 9.61 MB/sec execute 598 sec latency 90.833 ms 2024-09-17T09:49:03.914 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:03 smithi078 ceph-mon[63787]: Health check failed: Degraded data redundancy: 110/1512 objects degraded (7.275%), 18 pgs degraded (PG_DEGRADED) 2024-09-17T09:49:03.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:03 smithi136 ceph-mon[49560]: Health check failed: Degraded data redundancy: 110/1512 objects degraded (7.275%), 18 pgs degraded (PG_DEGRADED) 2024-09-17T09:49:03.976 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:03 smithi136 podman[59260]: 2024-09-17 09:49:03.711762641 +0000 UTC m=+0.435092434 container init e44d804a844772a54858cbf67721f8afe84fbde8a27a09b2ea25a8b8c4e0e188 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-17T09:49:03.976 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:03 smithi136 podman[59260]: 2024-09-17 09:49:03.716789754 +0000 UTC m=+0.440119538 container start e44d804a844772a54858cbf67721f8afe84fbde8a27a09b2ea25a8b8c4e0e188 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.vendor=CentOS) 2024-09-17T09:49:03.976 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:03 smithi136 podman[59260]: 2024-09-17 09:49:03.820143418 +0000 UTC m=+0.543473201 container attach e44d804a844772a54858cbf67721f8afe84fbde8a27a09b2ea25a8b8c4e0e188 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux ) 2024-09-17T09:49:03.976 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:03 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7-activate[59290]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:49:03.976 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:03 smithi136 bash[59260]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:49:03.976 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:03 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7-activate[59290]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:49:03.976 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:03 smithi136 bash[59260]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T09:49:04.678 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:04 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7-activate[59290]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-17T09:49:04.678 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:04 smithi136 bash[59260]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-17T09:49:04.679 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:04 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7-activate[59290]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-17T09:49:04.679 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:04 smithi136 bash[59260]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-17T09:49:04.679 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:04 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7-activate[59290]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-17T09:49:04.679 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:04 smithi136 bash[59260]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-17T09:49:04.864 INFO:teuthology.orchestra.run.smithi078.stdout: 5 287411 9.60 MB/sec execute 599 sec latency 55.448 ms 2024-09-17T09:49:04.964 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:04 smithi078 ceph-mon[63787]: pgmap v297: 97 pgs: 5 active+undersized, 7 peering, 30 active+undersized+degraded, 55 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 3.8 MiB/s wr, 436 op/s; 199/1509 objects degraded (13.188%); 770 B/s, 0 objects/s recovering 2024-09-17T09:49:04.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:04 smithi136 ceph-mon[49560]: pgmap v297: 97 pgs: 5 active+undersized, 7 peering, 30 active+undersized+degraded, 55 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 3.8 MiB/s wr, 436 op/s; 199/1509 objects degraded (13.188%); 770 B/s, 0 objects/s recovering 2024-09-17T09:49:04.977 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:04 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7-activate[59290]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-17T09:49:04.977 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:04 smithi136 bash[59260]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-17T09:49:04.977 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:04 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7-activate[59290]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-17T09:49:04.977 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:04 smithi136 bash[59260]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-17T09:49:04.977 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:04 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7-activate[59290]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-17T09:49:04.977 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:04 smithi136 bash[59260]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-17T09:49:04.977 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:04 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7-activate[59290]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-17T09:49:04.977 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:04 smithi136 bash[59260]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-17T09:49:04.977 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:04 smithi136 podman[59260]: 2024-09-17 09:49:04.715321952 +0000 UTC m=+1.438651745 container died e44d804a844772a54858cbf67721f8afe84fbde8a27a09b2ea25a8b8c4e0e188 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-17T09:49:05.662 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:05 smithi136 podman[59425]: 2024-09-17 09:49:05.409166702 +0000 UTC m=+0.682561053 container remove e44d804a844772a54858cbf67721f8afe84fbde8a27a09b2ea25a8b8c4e0e188 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7-activate, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-17T09:49:05.662 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:05 smithi136 podman[59481]: 2024-09-17 09:49:05.538071647 +0000 UTC m=+0.018355668 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:49:05.662 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:05 smithi136 podman[59481]: 2024-09-17 09:49:05.66048487 +0000 UTC m=+0.140768890 container create 6c9da904d39c62fe16c5011380e5067007db82e889f0a654e23f5cdaec5ebb99 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:49:05.864 INFO:teuthology.orchestra.run.smithi078.stdout: 5 cleanup 600 sec 2024-09-17T09:49:06.183 INFO:teuthology.orchestra.run.smithi078.stdout: 0 cleanup 600 sec 2024-09-17T09:49:06.184 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-17T09:49:06.184 INFO:teuthology.orchestra.run.smithi078.stdout: Operation Count AvgLat MaxLat 2024-09-17T09:49:06.184 INFO:teuthology.orchestra.run.smithi078.stdout: ---------------------------------------- 2024-09-17T09:49:06.184 INFO:teuthology.orchestra.run.smithi078.stdout: NTCreateX 184951 4.574 1297.993 2024-09-17T09:49:06.184 INFO:teuthology.orchestra.run.smithi078.stdout: Close 135762 0.131 18.784 2024-09-17T09:49:06.184 INFO:teuthology.orchestra.run.smithi078.stdout: Rename 7847 9.274 186.006 2024-09-17T09:49:06.184 INFO:teuthology.orchestra.run.smithi078.stdout: Unlink 37423 7.125 1301.749 2024-09-17T09:49:06.184 INFO:teuthology.orchestra.run.smithi078.stdout: Qpathinfo 167742 1.156 1101.030 2024-09-17T09:49:06.184 INFO:teuthology.orchestra.run.smithi078.stdout: Qfileinfo 29249 0.006 1.256 2024-09-17T09:49:06.184 INFO:teuthology.orchestra.run.smithi078.stdout: Qfsinfo 30765 0.837 1782.046 2024-09-17T09:49:06.184 INFO:teuthology.orchestra.run.smithi078.stdout: Sfileinfo 15097 13.122 1296.983 2024-09-17T09:49:06.184 INFO:teuthology.orchestra.run.smithi078.stdout: Find 64821 0.926 12.281 2024-09-17T09:49:06.184 INFO:teuthology.orchestra.run.smithi078.stdout: WriteX 91437 14.358 3158.645 2024-09-17T09:49:06.185 INFO:teuthology.orchestra.run.smithi078.stdout: ReadX 290024 0.010 4.118 2024-09-17T09:49:06.185 INFO:teuthology.orchestra.run.smithi078.stdout: LockX 606 0.360 3.714 2024-09-17T09:49:06.185 INFO:teuthology.orchestra.run.smithi078.stdout: UnlockX 606 0.312 2.485 2024-09-17T09:49:06.185 INFO:teuthology.orchestra.run.smithi078.stdout: Flush 12975 0.002 0.022 2024-09-17T09:49:06.185 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-17T09:49:06.185 INFO:teuthology.orchestra.run.smithi078.stdout:Throughput 9.59953 MB/sec 5 clients 5 procs max_latency=3158.654 ms 2024-09-17T09:49:06.186 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-09-17T09:49:06.221 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-09-17T09:49:06.225 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:05 smithi136 podman[59481]: 2024-09-17 09:49:05.938273217 +0000 UTC m=+0.418557229 container init 6c9da904d39c62fe16c5011380e5067007db82e889f0a654e23f5cdaec5ebb99 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-17T09:49:06.225 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:05 smithi136 podman[59481]: 2024-09-17 09:49:05.941677713 +0000 UTC m=+0.421961726 container start 6c9da904d39c62fe16c5011380e5067007db82e889f0a654e23f5cdaec5ebb99 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731) 2024-09-17T09:49:06.226 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:05 smithi136 ceph-osd[59500]: -- 172.21.15.136:0/3675665511 <== mon.1 v2:172.21.15.136:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x555d34f4e1e0 con 0x555d34eda000 2024-09-17T09:49:06.226 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:06 smithi136 bash[59481]: 6c9da904d39c62fe16c5011380e5067007db82e889f0a654e23f5cdaec5ebb99 2024-09-17T09:49:06.226 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:06 smithi136 systemd[1]: Started Ceph osd.7 for 35c0cd22-74d7-11ef-bceb-c7b262605968. 2024-09-17T09:49:06.340 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-09-17T09:49:06.364 INFO:teuthology.orchestra.run.smithi078.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-17T09:49:06.364 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-17T09:49:06.364 INFO:teuthology.orchestra.run.smithi078.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-09-17T09:49:06.365 INFO:teuthology.orchestra.run.smithi078.stdout:4 of 5 processes prepared for launch 0 sec 2024-09-17T09:49:06.465 INFO:teuthology.orchestra.run.smithi078.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-17T09:49:06.465 INFO:teuthology.orchestra.run.smithi078.stdout:releasing clients 2024-09-17T09:49:07.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:07 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:07.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:07 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:07.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:07 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:49:07.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:07 smithi078 ceph-mon[63787]: pgmap v298: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.5 MiB/s wr, 241 op/s; 245/1575 objects degraded (15.556%); 770 B/s, 0 objects/s recovering 2024-09-17T09:49:07.476 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:07 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:49:07.323+0000 7ffaa0ae0640 -1 Falling back to public interface 2024-09-17T09:49:07.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:07 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:07.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:07 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:07.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:07 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:49:07.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:07 smithi136 ceph-mon[49560]: pgmap v298: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.5 MiB/s wr, 241 op/s; 245/1575 objects degraded (15.556%); 770 B/s, 0 objects/s recovering 2024-09-17T09:49:08.466 INFO:teuthology.orchestra.run.smithi078.stdout: 5 370 22.37 MB/sec execute 1 sec latency 176.067 ms 2024-09-17T09:49:08.917 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:08 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:08.917 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:08 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:49:08.918 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:08 smithi078 ceph-mon[63787]: pgmap v299: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 361 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.5 MiB/s wr, 417 op/s; 253/1671 objects degraded (15.141%); 711 B/s, 0 objects/s recovering 2024-09-17T09:49:08.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:08 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:08.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:08 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:49:08.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:08 smithi136 ceph-mon[49560]: pgmap v299: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 361 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.5 MiB/s wr, 417 op/s; 253/1671 objects degraded (15.141%); 711 B/s, 0 objects/s recovering 2024-09-17T09:49:09.465 INFO:teuthology.orchestra.run.smithi078.stdout: 5 714 31.26 MB/sec execute 2 sec latency 55.111 ms 2024-09-17T09:49:09.922 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:09 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 253/1671 objects degraded (15.141%), 35 pgs degraded (PG_DEGRADED) 2024-09-17T09:49:10.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:09 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 253/1671 objects degraded (15.141%), 35 pgs degraded (PG_DEGRADED) 2024-09-17T09:49:10.465 INFO:teuthology.orchestra.run.smithi078.stdout: 5 1501 28.89 MB/sec execute 3 sec latency 15.088 ms 2024-09-17T09:49:10.927 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:10 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:10.928 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:10 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:10.928 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:10 smithi078 ceph-mon[63787]: pgmap v300: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 360 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 6.5 MiB/s wr, 382 op/s; 240/1599 objects degraded (15.009%); 616 B/s, 0 objects/s recovering 2024-09-17T09:49:10.928 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:10 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:10.928 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:10 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:11.196 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:10 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:11.196 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:10 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:11.196 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:10 smithi136 ceph-mon[49560]: pgmap v300: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 360 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 6.5 MiB/s wr, 382 op/s; 240/1599 objects degraded (15.009%); 616 B/s, 0 objects/s recovering 2024-09-17T09:49:11.196 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:10 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:11.196 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:10 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:11.465 INFO:teuthology.orchestra.run.smithi078.stdout: 5 2254 23.09 MB/sec execute 4 sec latency 32.806 ms 2024-09-17T09:49:12.465 INFO:teuthology.orchestra.run.smithi078.stdout: 5 cleanup 5 sec 2024-09-17T09:49:12.686 INFO:teuthology.orchestra.run.smithi078.stdout: 0 cleanup 5 sec 2024-09-17T09:49:12.687 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-17T09:49:12.687 INFO:teuthology.orchestra.run.smithi078.stdout: Operation Count AvgLat MaxLat 2024-09-17T09:49:12.687 INFO:teuthology.orchestra.run.smithi078.stdout: ---------------------------------------- 2024-09-17T09:49:12.687 INFO:teuthology.orchestra.run.smithi078.stdout: NTCreateX 1943 3.806 62.397 2024-09-17T09:49:12.687 INFO:teuthology.orchestra.run.smithi078.stdout: Close 1691 3.254 58.574 2024-09-17T09:49:12.687 INFO:teuthology.orchestra.run.smithi078.stdout: Rename 76 7.816 43.885 2024-09-17T09:49:12.687 INFO:teuthology.orchestra.run.smithi078.stdout: Unlink 187 3.438 20.085 2024-09-17T09:49:12.687 INFO:teuthology.orchestra.run.smithi078.stdout: Qpathinfo 1680 1.401 32.793 2024-09-17T09:49:12.687 INFO:teuthology.orchestra.run.smithi078.stdout: Qfileinfo 571 8.562 60.985 2024-09-17T09:49:12.687 INFO:teuthology.orchestra.run.smithi078.stdout: Qfsinfo 212 0.640 5.469 2024-09-17T09:49:12.687 INFO:teuthology.orchestra.run.smithi078.stdout: Sfileinfo 241 7.602 27.150 2024-09-17T09:49:12.687 INFO:teuthology.orchestra.run.smithi078.stdout: Find 572 0.920 3.947 2024-09-17T09:49:12.687 INFO:teuthology.orchestra.run.smithi078.stdout: WriteX 2182 0.022 0.085 2024-09-17T09:49:12.687 INFO:teuthology.orchestra.run.smithi078.stdout: ReadX 2053 0.014 0.790 2024-09-17T09:49:12.688 INFO:teuthology.orchestra.run.smithi078.stdout: Flush 170 6.427 22.680 2024-09-17T09:49:12.688 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-17T09:49:12.688 INFO:teuthology.orchestra.run.smithi078.stdout:Throughput 23.0924 MB/sec 5 clients 5 procs max_latency=176.067 ms 2024-09-17T09:49:12.689 DEBUG:teuthology.parallel:result is None 2024-09-17T09:49:13.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:13 smithi136 ceph-mon[49560]: pgmap v301: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 355 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 12 MiB/s wr, 531 op/s; 235/1593 objects degraded (14.752%); 528 B/s, 0 objects/s recovering 2024-09-17T09:49:13.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:13 smithi078 ceph-mon[63787]: pgmap v301: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 355 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 12 MiB/s wr, 531 op/s; 235/1593 objects degraded (14.752%); 528 B/s, 0 objects/s recovering 2024-09-17T09:49:15.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:15.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:15.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:49:15.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:49:15.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:15.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:15.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:15 smithi136 ceph-mon[49560]: pgmap v302: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 12 MiB/s wr, 550 op/s; 231/1491 objects degraded (15.493%); 513 B/s, 0 objects/s recovering 2024-09-17T09:49:15.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:49:15.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:15.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:15.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:15.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:15.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:15.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-17T09:49:15.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-17T09:49:15.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-17T09:49:15.477 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-17T09:49:15.478 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-17T09:49:15.478 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-17T09:49:15.478 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-17T09:49:15.478 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-17T09:49:15.478 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-17T09:49:15.478 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-17T09:49:15.478 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-17T09:49:15.478 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-17T09:49:15.478 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-17T09:49:15.478 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-17T09:49:15.478 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-17T09:49:15.478 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-17T09:49:15.478 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-17T09:49:15.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:15.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:15.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:49:15.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:49:15.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:15.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:15.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:15 smithi078 ceph-mon[63787]: pgmap v302: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 12 MiB/s wr, 550 op/s; 231/1491 objects degraded (15.493%); 513 B/s, 0 objects/s recovering 2024-09-17T09:49:15.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:49:15.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:15.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:15.598 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:15.598 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:15.598 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:15.598 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-17T09:49:15.598 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-17T09:49:15.598 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-17T09:49:15.598 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-17T09:49:15.598 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-17T09:49:15.598 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-17T09:49:15.598 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-17T09:49:15.598 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-17T09:49:15.598 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-17T09:49:15.598 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-17T09:49:15.598 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-17T09:49:15.598 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-17T09:49:15.599 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-17T09:49:15.599 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-17T09:49:15.599 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-17T09:49:15.599 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-17T09:49:15.599 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-17T09:49:16.323 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:16 smithi078 ceph-mon[63787]: Upgrade: Setting container_image for all osd 2024-09-17T09:49:16.323 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:16 smithi078 ceph-mon[63787]: Upgrade: Setting require_osd_release to 19 squid 2024-09-17T09:49:16.323 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:16 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 231/1491 objects degraded (15.493%), 35 pgs degraded (PG_DEGRADED) 2024-09-17T09:49:16.323 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:16 smithi078 ceph-mon[63787]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-17T09:49:16.323 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:16 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-17T09:49:16.323 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:16 smithi078 ceph-mon[63787]: osdmap e152: 8 total, 7 up, 8 in 2024-09-17T09:49:16.323 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:16 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi078.jdrsup"]} : dispatch 2024-09-17T09:49:16.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:16 smithi136 ceph-mon[49560]: Upgrade: Setting container_image for all osd 2024-09-17T09:49:16.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:16 smithi136 ceph-mon[49560]: Upgrade: Setting require_osd_release to 19 squid 2024-09-17T09:49:16.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:16 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 231/1491 objects degraded (15.493%), 35 pgs degraded (PG_DEGRADED) 2024-09-17T09:49:16.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:16 smithi136 ceph-mon[49560]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-17T09:49:16.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:16 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-17T09:49:16.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:16 smithi136 ceph-mon[49560]: osdmap e152: 8 total, 7 up, 8 in 2024-09-17T09:49:16.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:16 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi078.jdrsup"]} : dispatch 2024-09-17T09:49:17.225 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:16 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:49:16.856+0000 7ffaa0ae0640 -1 osd.7 0 read_superblock omap replica is missing. 2024-09-17T09:49:17.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:17 smithi078 ceph-mon[63787]: Upgrade: It appears safe to stop mds.foofs.smithi078.jdrsup 2024-09-17T09:49:17.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:17 smithi078 ceph-mon[63787]: pgmap v304: 97 pgs: 11 active+undersized, 31 active+undersized+degraded, 55 active+clean; 291 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 14 MiB/s wr, 582 op/s; 136/999 objects degraded (13.614%) 2024-09-17T09:49:17.599 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-17T09:49:17.674 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:17 smithi136 ceph-mon[49560]: Upgrade: It appears safe to stop mds.foofs.smithi078.jdrsup 2024-09-17T09:49:17.675 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:17 smithi136 ceph-mon[49560]: pgmap v304: 97 pgs: 11 active+undersized, 31 active+undersized+degraded, 55 active+clean; 291 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 14 MiB/s wr, 582 op/s; 136/999 objects degraded (13.614%) 2024-09-17T09:49:18.311 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:49:18.311 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (9m) 2m ago 18m 17.8M - 0.27.0 11f11916f8cd 97db2373ce92 2024-09-17T09:49:18.311 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (6m) 2m ago 18m 6899k - 19.3.0-4963-ge646f729 f271290a0950 da528c5ca445 2024-09-17T09:49:18.311 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi136 smithi136 running (6m) 8s ago 18m 6899k - 19.3.0-4963-ge646f729 f271290a0950 909f405348cc 2024-09-17T09:49:18.311 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (9m) 2m ago 18m 53.0M - dad864ee21e9 db74b0f8e0a7 2024-09-17T09:49:18.311 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.jdrsup smithi078 running (12m) 2m ago 12m 38.4M - 17.2.0 e1d6a67b021e 8f834e15ba88 2024-09-17T09:49:18.311 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi136.smslff smithi136 running (12m) 8s ago 12m 17.0M - 17.2.0 e1d6a67b021e 0c20e2b328d3 2024-09-17T09:49:18.311 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rsqofi smithi078 *:8443,9283,8765 running (8m) 2m ago 20m 519M - 19.3.0-4963-ge646f729 f271290a0950 7dda3108c9a0 2024-09-17T09:49:18.311 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi136.hicmlm smithi136 *:8443,9283,8765 running (7m) 8s ago 18m 456M - 19.3.0-4963-ge646f729 f271290a0950 fc518b58e07a 2024-09-17T09:49:18.311 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (7m) 2m ago 20m 53.2M 2048M 19.3.0-4963-ge646f729 f271290a0950 3ead101d3db1 2024-09-17T09:49:18.311 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi136 smithi136 running (7m) 8s ago 18m 46.7M 2048M 19.3.0-4963-ge646f729 f271290a0950 4fce88842b79 2024-09-17T09:49:18.311 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.tcwbvi smithi078 *:2049 running (12m) 2m ago 12m 284M - 4.0 e1d6a67b021e 30c87cb46555 2024-09-17T09:49:18.312 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi136.regjvc smithi136 *:2049 running (12m) 8s ago 12m 90.3M - 4.0 e1d6a67b021e b88e4dc24376 2024-09-17T09:49:18.312 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (9m) 2m ago 18m 11.0M - 1.7.0 72c9c2088986 52dad3adeeaa 2024-09-17T09:49:18.312 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi136 smithi136 *:9100 running (8m) 8s ago 17m 10.6M - 1.7.0 72c9c2088986 f2bc5856e35a 2024-09-17T09:49:18.312 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (5m) 2m ago 17m 351M 1936M 19.3.0-4963-ge646f729 f271290a0950 1ea81cc809b8 2024-09-17T09:49:18.312 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (4m) 2m ago 16m 290M 1936M 19.3.0-4963-ge646f729 f271290a0950 eda32560334b 2024-09-17T09:49:18.312 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (3m) 2m ago 16m 213M 1936M 19.3.0-4963-ge646f729 f271290a0950 83a32ad7c358 2024-09-17T09:49:18.312 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (3m) 2m ago 15m 14.3M 1936M 19.3.0-4963-ge646f729 f271290a0950 e2cd18998d2b 2024-09-17T09:49:18.312 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi136 running (2m) 8s ago 15m 354M 2704M 19.3.0-4963-ge646f729 f271290a0950 e2a18ad94664 2024-09-17T09:49:18.312 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi136 running (100s) 8s ago 15m 310M 2704M 19.3.0-4963-ge646f729 f271290a0950 df9b6954ce0f 2024-09-17T09:49:18.312 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi136 running (48s) 8s ago 14m 284M 2704M 19.3.0-4963-ge646f729 f271290a0950 c477e7043301 2024-09-17T09:49:18.312 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi136 running (12s) 8s ago 14m 11.2M 2704M 19.3.0-4963-ge646f729 f271290a0950 6c9da904d39c 2024-09-17T09:49:18.312 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (8m) 2m ago 17m 53.8M - 2.51.0 1d3b7f56885b 6e5e8c721eb0 2024-09-17T09:49:18.725 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:18 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:49:18.280+0000 7ffaa0ae0640 -1 osd.7 149 log_to_monitors true 2024-09-17T09:49:18.746 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:49:18.746 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-17T09:49:18.746 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:49:18.747 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:49:18.747 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-17T09:49:18.747 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:49:18.747 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:49:18.747 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-17T09:49:18.747 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 7 2024-09-17T09:49:18.747 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:49:18.747 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-17T09:49:18.747 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T09:49:18.747 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:49:18.747 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-17T09:49:18.747 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-17T09:49:18.747 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 11 2024-09-17T09:49:18.747 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-17T09:49:18.748 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:49:19.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:18 smithi078 ceph-mon[63787]: from='client.34456 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:49:19.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:18 smithi078 ceph-mon[63787]: Upgrade: Updating mds.foofs.smithi078.jdrsup 2024-09-17T09:49:19.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:19.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.jdrsup", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-17T09:49:19.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:49:19.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:18 smithi078 ceph-mon[63787]: Deploying daemon mds.foofs.smithi078.jdrsup on smithi078 2024-09-17T09:49:19.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:18 smithi078 ceph-mon[63787]: from='client.34460 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:49:19.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:18 smithi078 ceph-mon[63787]: from='osd.7 [v2:172.21.15.136:6824/3025091295,v1:172.21.15.136:6825/3025091295]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-17T09:49:19.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:18 smithi078 ceph-mon[63787]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-17T09:49:19.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:18 smithi078 ceph-mon[63787]: from='client.34464 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:49:19.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:18 smithi078 ceph-mon[63787]: pgmap v305: 97 pgs: 14 active+undersized, 28 active+undersized+degraded, 55 active+clean; 240 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 11 MiB/s wr, 442 op/s; 87/498 objects degraded (17.470%) 2024-09-17T09:49:19.099 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:49:19.099 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407", 2024-09-17T09:49:19.099 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-17T09:49:19.099 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T09:49:19.099 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-17T09:49:19.099 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-09-17T09:49:19.099 INFO:teuthology.orchestra.run.smithi078.stdout: "osd", 2024-09-17T09:49:19.099 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-17T09:49:19.099 INFO:teuthology.orchestra.run.smithi078.stdout: "mon" 2024-09-17T09:49:19.099 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-17T09:49:19.099 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "14/23 daemons upgraded", 2024-09-17T09:49:19.100 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading mds daemons", 2024-09-17T09:49:19.100 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-17T09:49:19.100 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:49:19.226 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:49:18 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:49:18.739+0000 7ffa9888a640 -1 osd.7 149 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T09:49:19.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:18 smithi136 ceph-mon[49560]: from='client.34456 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:49:19.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:18 smithi136 ceph-mon[49560]: Upgrade: Updating mds.foofs.smithi078.jdrsup 2024-09-17T09:49:19.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:19.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.jdrsup", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-17T09:49:19.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:49:19.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:18 smithi136 ceph-mon[49560]: Deploying daemon mds.foofs.smithi078.jdrsup on smithi078 2024-09-17T09:49:19.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:18 smithi136 ceph-mon[49560]: from='client.34460 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:49:19.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:18 smithi136 ceph-mon[49560]: from='osd.7 [v2:172.21.15.136:6824/3025091295,v1:172.21.15.136:6825/3025091295]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-17T09:49:19.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:18 smithi136 ceph-mon[49560]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-17T09:49:19.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:18 smithi136 ceph-mon[49560]: from='client.34464 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:49:19.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:18 smithi136 ceph-mon[49560]: pgmap v305: 97 pgs: 14 active+undersized, 28 active+undersized+degraded, 55 active+clean; 240 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 11 MiB/s wr, 442 op/s; 87/498 objects degraded (17.470%) 2024-09-17T09:49:19.528 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 87/498 objects degraded (17.470%), 28 pgs degraded 2024-09-17T09:49:19.528 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-17T09:49:19.528 INFO:teuthology.orchestra.run.smithi078.stdout: osd.7 (root=default,host=smithi136) is down 2024-09-17T09:49:19.528 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 87/498 objects degraded (17.470%), 28 pgs degraded 2024-09-17T09:49:19.528 INFO:teuthology.orchestra.run.smithi078.stdout: pg 1.0 is active+undersized+degraded, acting [0,6] 2024-09-17T09:49:19.529 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.0 is active+undersized+degraded, acting [1,0] 2024-09-17T09:49:19.529 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.4 is active+undersized+degraded, acting [1,0] 2024-09-17T09:49:19.529 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.5 is active+undersized+degraded, acting [0,4] 2024-09-17T09:49:19.532 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.7 is active+undersized+degraded, acting [6,2] 2024-09-17T09:49:19.532 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.8 is active+undersized+degraded, acting [0,1] 2024-09-17T09:49:19.532 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.9 is active+undersized+degraded, acting [1,3] 2024-09-17T09:49:19.533 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.a is active+undersized+degraded, acting [1,3] 2024-09-17T09:49:19.533 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.b is active+undersized+degraded, acting [4,5] 2024-09-17T09:49:19.533 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.e is active+undersized+degraded, acting [2,3] 2024-09-17T09:49:19.533 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.f is active+undersized+degraded, acting [4,0] 2024-09-17T09:49:19.533 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.12 is active+undersized+degraded, acting [5,3] 2024-09-17T09:49:19.533 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.18 is active+undersized+degraded, acting [5,4] 2024-09-17T09:49:19.533 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1a is active+undersized+degraded, acting [6,4] 2024-09-17T09:49:19.533 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1b is active+undersized+degraded, acting [3,6] 2024-09-17T09:49:19.533 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1c is active+undersized+degraded, acting [5,2] 2024-09-17T09:49:19.533 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1d is active+undersized+degraded, acting [6,0] 2024-09-17T09:49:19.533 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1f is active+undersized+degraded, acting [0,4] 2024-09-17T09:49:19.533 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.d is active+undersized+degraded, acting [5,6] 2024-09-17T09:49:19.533 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.e is active+undersized+degraded, acting [4,1] 2024-09-17T09:49:19.534 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.12 is active+undersized+degraded, acting [0,3] 2024-09-17T09:49:19.534 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.14 is active+undersized+degraded, acting [4,6] 2024-09-17T09:49:19.534 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.16 is active+undersized+degraded, acting [5,1] 2024-09-17T09:49:19.534 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1b is active+undersized+degraded, acting [0,4] 2024-09-17T09:49:19.534 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.0 is active+undersized+degraded, acting [3,0] 2024-09-17T09:49:19.534 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.5 is active+undersized+degraded, acting [6,4] 2024-09-17T09:49:19.534 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.a is active+undersized+degraded, acting [6,1] 2024-09-17T09:49:19.534 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.14 is active+undersized+degraded, acting [3,2] 2024-09-17T09:49:19.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:19 smithi078 ceph-mon[63787]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-17T09:49:19.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:19 smithi078 ceph-mon[63787]: from='osd.7 [v2:172.21.15.136:6824/3025091295,v1:172.21.15.136:6825/3025091295]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi136", "root=default"]} : dispatch 2024-09-17T09:49:19.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:19 smithi078 ceph-mon[63787]: osdmap e153: 8 total, 7 up, 8 in 2024-09-17T09:49:19.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:19 smithi078 ceph-mon[63787]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi136", "root=default"]} : dispatch 2024-09-17T09:49:19.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:19 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/3203644157' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:19.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:19 smithi078 ceph-mon[63787]: from='client.34472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:49:19.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:19 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/3882867104' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:49:20.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:19 smithi136 ceph-mon[49560]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-17T09:49:20.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:19 smithi136 ceph-mon[49560]: from='osd.7 [v2:172.21.15.136:6824/3025091295,v1:172.21.15.136:6825/3025091295]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi136", "root=default"]} : dispatch 2024-09-17T09:49:20.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:19 smithi136 ceph-mon[49560]: osdmap e153: 8 total, 7 up, 8 in 2024-09-17T09:49:20.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:19 smithi136 ceph-mon[49560]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi136", "root=default"]} : dispatch 2024-09-17T09:49:20.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:19 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/3203644157' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:20.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:19 smithi136 ceph-mon[49560]: from='client.34472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:49:20.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:19 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/3882867104' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:49:20.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:20 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi078[63783]: 2024-09-17T09:49:20.278+0000 7f49af00c640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T09:49:21.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:20 smithi078 ceph-mon[63787]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:49:21.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:20 smithi078 ceph-mon[63787]: osd.7 [v2:172.21.15.136:6824/3025091295,v1:172.21.15.136:6825/3025091295] boot 2024-09-17T09:49:21.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:20 smithi078 ceph-mon[63787]: osdmap e154: 8 total, 8 up, 8 in 2024-09-17T09:49:21.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:20 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-17T09:49:21.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:20 smithi078 ceph-mon[63787]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-17T09:49:21.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:20 smithi078 ceph-mon[63787]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T09:49:21.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:20 smithi078 ceph-mon[63787]: osdmap e155: 8 total, 8 up, 8 in 2024-09-17T09:49:21.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:20 smithi078 ceph-mon[63787]: Standby daemon mds.foofs.smithi136.smslff assigned to filesystem foofs as rank 0 2024-09-17T09:49:21.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:20 smithi078 ceph-mon[63787]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-17T09:49:21.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:20 smithi078 ceph-mon[63787]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-17T09:49:21.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:20 smithi078 ceph-mon[63787]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-17T09:49:21.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:20 smithi078 ceph-mon[63787]: fsmap foofs:1/1 {0=foofs.smithi136.smslff=up:replay} 2024-09-17T09:49:21.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:20 smithi078 ceph-mon[63787]: pgmap v309: 97 pgs: 16 active+undersized, 26 active+undersized+degraded, 55 active+clean; 235 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 910 KiB/s wr, 121 op/s; 80/438 objects degraded (18.265%) 2024-09-17T09:49:21.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:20 smithi136 ceph-mon[49560]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T09:49:21.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:20 smithi136 ceph-mon[49560]: osd.7 [v2:172.21.15.136:6824/3025091295,v1:172.21.15.136:6825/3025091295] boot 2024-09-17T09:49:21.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:20 smithi136 ceph-mon[49560]: osdmap e154: 8 total, 8 up, 8 in 2024-09-17T09:49:21.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:20 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-17T09:49:21.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:20 smithi136 ceph-mon[49560]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-17T09:49:21.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:20 smithi136 ceph-mon[49560]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T09:49:21.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:20 smithi136 ceph-mon[49560]: osdmap e155: 8 total, 8 up, 8 in 2024-09-17T09:49:21.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:20 smithi136 ceph-mon[49560]: Standby daemon mds.foofs.smithi136.smslff assigned to filesystem foofs as rank 0 2024-09-17T09:49:21.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:20 smithi136 ceph-mon[49560]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-17T09:49:21.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:20 smithi136 ceph-mon[49560]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-17T09:49:21.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:20 smithi136 ceph-mon[49560]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-17T09:49:21.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:20 smithi136 ceph-mon[49560]: fsmap foofs:1/1 {0=foofs.smithi136.smslff=up:replay} 2024-09-17T09:49:21.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:20 smithi136 ceph-mon[49560]: pgmap v309: 97 pgs: 16 active+undersized, 26 active+undersized+degraded, 55 active+clean; 235 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 910 KiB/s wr, 121 op/s; 80/438 objects degraded (18.265%) 2024-09-17T09:49:21.725 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:49:21 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:49:21.320+0000 7f9075bb7640 -1 bdev(0x5600e2036c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 1 2024-09-17T09:49:21.726 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:49:21 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:49:21.321+0000 7f9072bb1640 -1 bdev(0x5600e2036c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 3 2024-09-17T09:49:22.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:22 smithi078 ceph-mon[63787]: osdmap e156: 8 total, 8 up, 8 in 2024-09-17T09:49:22.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:22 smithi078 ceph-mon[63787]: Health check update: Degraded data redundancy: 80/438 objects degraded (18.265%), 26 pgs degraded (PG_DEGRADED) 2024-09-17T09:49:22.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:22 smithi136 ceph-mon[49560]: osdmap e156: 8 total, 8 up, 8 in 2024-09-17T09:49:22.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:22 smithi136 ceph-mon[49560]: Health check update: Degraded data redundancy: 80/438 objects degraded (18.265%), 26 pgs degraded (PG_DEGRADED) 2024-09-17T09:49:23.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:23 smithi078 ceph-mon[63787]: reconnect by client.24321 172.21.15.136:0/3941243321 after 0 2024-09-17T09:49:23.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:23 smithi078 ceph-mon[63787]: reconnect by client.14548 172.21.15.78:0/3270440936 after 0 2024-09-17T09:49:23.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:23 smithi078 ceph-mon[63787]: mds.? [v2:172.21.15.136:6832/1260769482,v1:172.21.15.136:6833/1260769482] up:reconnect 2024-09-17T09:49:23.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:23 smithi078 ceph-mon[63787]: fsmap foofs:1/1 {0=foofs.smithi136.smslff=up:reconnect} 2024-09-17T09:49:23.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:23 smithi078 ceph-mon[63787]: osdmap e157: 8 total, 8 up, 8 in 2024-09-17T09:49:23.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:23 smithi078 ceph-mon[63787]: pgmap v312: 97 pgs: 1 active+undersized+remapped, 2 remapped+peering, 13 active+undersized, 13 active+undersized+degraded, 68 active+clean; 233 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 58 KiB/s wr, 16 op/s; 40/432 objects degraded (9.259%); 930 KiB/s, 16 objects/s recovering 2024-09-17T09:49:23.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:23 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:23.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:23 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:49:23.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:23 smithi136 ceph-mon[49560]: reconnect by client.24321 172.21.15.136:0/3941243321 after 0 2024-09-17T09:49:23.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:23 smithi136 ceph-mon[49560]: reconnect by client.14548 172.21.15.78:0/3270440936 after 0 2024-09-17T09:49:23.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:23 smithi136 ceph-mon[49560]: mds.? [v2:172.21.15.136:6832/1260769482,v1:172.21.15.136:6833/1260769482] up:reconnect 2024-09-17T09:49:23.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:23 smithi136 ceph-mon[49560]: fsmap foofs:1/1 {0=foofs.smithi136.smslff=up:reconnect} 2024-09-17T09:49:23.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:23 smithi136 ceph-mon[49560]: osdmap e157: 8 total, 8 up, 8 in 2024-09-17T09:49:23.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:23 smithi136 ceph-mon[49560]: pgmap v312: 97 pgs: 1 active+undersized+remapped, 2 remapped+peering, 13 active+undersized, 13 active+undersized+degraded, 68 active+clean; 233 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 58 KiB/s wr, 16 op/s; 40/432 objects degraded (9.259%); 930 KiB/s, 16 objects/s recovering 2024-09-17T09:49:23.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:23 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:23.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:23 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:49:24.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:24 smithi078 ceph-mon[63787]: mds.? [v2:172.21.15.136:6832/1260769482,v1:172.21.15.136:6833/1260769482] up:rejoin 2024-09-17T09:49:24.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:24 smithi078 ceph-mon[63787]: fsmap foofs:1/1 {0=foofs.smithi136.smslff=up:rejoin} 2024-09-17T09:49:24.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:24 smithi078 ceph-mon[63787]: osdmap e158: 8 total, 8 up, 8 in 2024-09-17T09:49:24.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:24 smithi078 ceph-mon[63787]: daemon mds.foofs.smithi136.smslff is now active in filesystem foofs as rank 0 2024-09-17T09:49:24.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:24 smithi136 ceph-mon[49560]: mds.? [v2:172.21.15.136:6832/1260769482,v1:172.21.15.136:6833/1260769482] up:rejoin 2024-09-17T09:49:24.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:24 smithi136 ceph-mon[49560]: fsmap foofs:1/1 {0=foofs.smithi136.smslff=up:rejoin} 2024-09-17T09:49:24.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:24 smithi136 ceph-mon[49560]: osdmap e158: 8 total, 8 up, 8 in 2024-09-17T09:49:24.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:24 smithi136 ceph-mon[49560]: daemon mds.foofs.smithi136.smslff is now active in filesystem foofs as rank 0 2024-09-17T09:49:25.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:25 smithi078 ceph-mon[63787]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-17T09:49:25.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:25 smithi078 ceph-mon[63787]: mds.? [v2:172.21.15.136:6832/1260769482,v1:172.21.15.136:6833/1260769482] up:active 2024-09-17T09:49:25.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:25 smithi078 ceph-mon[63787]: fsmap foofs:1 {0=foofs.smithi136.smslff=up:active} 2024-09-17T09:49:25.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:25 smithi078 ceph-mon[63787]: pgmap v314: 97 pgs: 3 peering, 1 active+undersized+remapped, 2 remapped+peering, 91 active+clean; 233 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 45 MiB/s rd, 52 KiB/s wr, 17 op/s; 976 KiB/s, 87 objects/s recovering 2024-09-17T09:49:25.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:25 smithi136 ceph-mon[49560]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-17T09:49:25.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:25 smithi136 ceph-mon[49560]: mds.? [v2:172.21.15.136:6832/1260769482,v1:172.21.15.136:6833/1260769482] up:active 2024-09-17T09:49:25.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:25 smithi136 ceph-mon[49560]: fsmap foofs:1 {0=foofs.smithi136.smslff=up:active} 2024-09-17T09:49:25.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:25 smithi136 ceph-mon[49560]: pgmap v314: 97 pgs: 3 peering, 1 active+undersized+remapped, 2 remapped+peering, 91 active+clean; 233 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 45 MiB/s rd, 52 KiB/s wr, 17 op/s; 976 KiB/s, 87 objects/s recovering 2024-09-17T09:49:26.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:26 smithi078 ceph-mon[63787]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 40/432 objects degraded (9.259%), 13 pgs degraded) 2024-09-17T09:49:26.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:26 smithi136 ceph-mon[49560]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 40/432 objects degraded (9.259%), 13 pgs degraded) 2024-09-17T09:49:27.503 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:27 smithi078 ceph-mon[63787]: pgmap v315: 97 pgs: 3 peering, 1 active+undersized+remapped, 93 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 38 MiB/s rd, 37 KiB/s wr, 13 op/s; 1.3 MiB/s, 69 objects/s recovering 2024-09-17T09:49:27.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:27 smithi136 ceph-mon[49560]: pgmap v315: 97 pgs: 3 peering, 1 active+undersized+remapped, 93 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 38 MiB/s rd, 37 KiB/s wr, 13 op/s; 1.3 MiB/s, 69 objects/s recovering 2024-09-17T09:49:28.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:28 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:28.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:28 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:28.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:28 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:49:28.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:28 smithi078 ceph-mon[63787]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-17T09:49:28.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:28 smithi078 ceph-mon[63787]: Cluster is now healthy 2024-09-17T09:49:28.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:28 smithi078 ceph-mon[63787]: mds.? [v2:172.21.15.78:6834/1849336238,v1:172.21.15.78:6835/1849336238] up:boot 2024-09-17T09:49:28.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:28 smithi078 ceph-mon[63787]: fsmap foofs:1 {0=foofs.smithi136.smslff=up:active} 1 up:standby 2024-09-17T09:49:28.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:28 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mds metadata", "who": "foofs.smithi078.jdrsup"} : dispatch 2024-09-17T09:49:28.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:28 smithi078 ceph-mon[63787]: pgmap v316: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 35 KiB/s wr, 16 op/s; 1.1 MiB/s, 49 objects/s recovering 2024-09-17T09:49:28.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:28 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:28.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:28 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:28.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:28 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:49:28.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:28 smithi136 ceph-mon[49560]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-17T09:49:28.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:28 smithi136 ceph-mon[49560]: Cluster is now healthy 2024-09-17T09:49:28.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:28 smithi136 ceph-mon[49560]: mds.? [v2:172.21.15.78:6834/1849336238,v1:172.21.15.78:6835/1849336238] up:boot 2024-09-17T09:49:28.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:28 smithi136 ceph-mon[49560]: fsmap foofs:1 {0=foofs.smithi136.smslff=up:active} 1 up:standby 2024-09-17T09:49:28.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:28 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mds metadata", "who": "foofs.smithi078.jdrsup"} : dispatch 2024-09-17T09:49:28.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:28 smithi136 ceph-mon[49560]: pgmap v316: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 35 KiB/s wr, 16 op/s; 1.1 MiB/s, 49 objects/s recovering 2024-09-17T09:49:31.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:31 smithi078 ceph-mon[63787]: pgmap v317: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 4.0 KiB/s wr, 12 op/s; 572 KiB/s, 43 objects/s recovering 2024-09-17T09:49:31.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:31 smithi136 ceph-mon[49560]: pgmap v317: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 4.0 KiB/s wr, 12 op/s; 572 KiB/s, 43 objects/s recovering 2024-09-17T09:49:33.668 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:33 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:33.668 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:33 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:33.668 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:33 smithi078 ceph-mon[63787]: pgmap v318: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 8.4 KiB/s wr, 10 op/s; 470 KiB/s, 35 objects/s recovering 2024-09-17T09:49:33.668 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:33 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:33.668 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:33 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:33.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:33 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:33.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:33 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:33.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:33 smithi136 ceph-mon[49560]: pgmap v318: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 8.4 KiB/s wr, 10 op/s; 470 KiB/s, 35 objects/s recovering 2024-09-17T09:49:33.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:33 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:33.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:33 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:35.693 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:35 smithi078 ceph-mon[63787]: pgmap v319: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 7.5 KiB/s wr, 8 op/s; 365 KiB/s, 25 objects/s recovering 2024-09-17T09:49:35.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:35 smithi136 ceph-mon[49560]: pgmap v319: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 7.5 KiB/s wr, 8 op/s; 365 KiB/s, 25 objects/s recovering 2024-09-17T09:49:37.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:37 smithi136 ceph-mon[49560]: Detected new or changed devices on smithi078 2024-09-17T09:49:37.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:37 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:37.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:37 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:37.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:37 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:49:37.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:37 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:49:37.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:37 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:37.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:37 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:37.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:37 smithi136 ceph-mon[49560]: pgmap v320: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 7.0 KiB/s wr, 6 op/s; 341 KiB/s, 3 objects/s recovering 2024-09-17T09:49:37.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:37 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:49:37.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:37 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:37.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:37 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:37.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:37 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:37.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:37 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:37.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:37 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi136.smslff"]} : dispatch 2024-09-17T09:49:37.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:37 smithi078 ceph-mon[63787]: Detected new or changed devices on smithi078 2024-09-17T09:49:37.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:37 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:37.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:37 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:37.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:37 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:49:37.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:37 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:49:37.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:37 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:37.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:37 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:37.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:37 smithi078 ceph-mon[63787]: pgmap v320: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 7.0 KiB/s wr, 6 op/s; 341 KiB/s, 3 objects/s recovering 2024-09-17T09:49:37.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:37 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:49:37.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:37 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:37.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:37 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:37.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:37 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:37.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:37 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:37.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:37 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi136.smslff"]} : dispatch 2024-09-17T09:49:38.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:38 smithi136 ceph-mon[49560]: Upgrade: It appears safe to stop mds.foofs.smithi136.smslff 2024-09-17T09:49:38.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:38.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:49:38.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:38.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:38.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi136.smslff", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-17T09:49:38.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:49:38.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:38 smithi078 ceph-mon[63787]: Upgrade: It appears safe to stop mds.foofs.smithi136.smslff 2024-09-17T09:49:38.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:38.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:49:38.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:38.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:38.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi136.smslff", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-17T09:49:38.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:49:39.671 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:39 smithi136 ceph-mon[49560]: Upgrade: Updating mds.foofs.smithi136.smslff 2024-09-17T09:49:39.672 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:39 smithi136 ceph-mon[49560]: Deploying daemon mds.foofs.smithi136.smslff on smithi136 2024-09-17T09:49:39.672 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:39 smithi136 ceph-mon[49560]: pgmap v321: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.7 KiB/s rd, 7.0 KiB/s wr, 5 op/s 2024-09-17T09:49:39.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:39 smithi078 ceph-mon[63787]: Upgrade: Updating mds.foofs.smithi136.smslff 2024-09-17T09:49:39.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:39 smithi078 ceph-mon[63787]: Deploying daemon mds.foofs.smithi136.smslff on smithi136 2024-09-17T09:49:39.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:39 smithi078 ceph-mon[63787]: pgmap v321: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.7 KiB/s rd, 7.0 KiB/s wr, 5 op/s 2024-09-17T09:49:40.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:39 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi078[63783]: 2024-09-17T09:49:39.935+0000 7f49af00c640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T09:49:40.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:40 smithi078 ceph-mon[63787]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-17T09:49:40.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:40 smithi078 ceph-mon[63787]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T09:49:40.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:40 smithi078 ceph-mon[63787]: osdmap e159: 8 total, 8 up, 8 in 2024-09-17T09:49:40.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:40 smithi078 ceph-mon[63787]: Standby daemon mds.foofs.smithi078.jdrsup assigned to filesystem foofs as rank 0 2024-09-17T09:49:40.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:40 smithi078 ceph-mon[63787]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-17T09:49:40.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:40 smithi078 ceph-mon[63787]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-17T09:49:40.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:40 smithi078 ceph-mon[63787]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-17T09:49:40.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:40 smithi078 ceph-mon[63787]: fsmap foofs:1/1 {0=foofs.smithi078.jdrsup=up:replay} 2024-09-17T09:49:40.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:40 smithi136 ceph-mon[49560]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-17T09:49:40.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:40 smithi136 ceph-mon[49560]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T09:49:40.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:40 smithi136 ceph-mon[49560]: osdmap e159: 8 total, 8 up, 8 in 2024-09-17T09:49:40.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:40 smithi136 ceph-mon[49560]: Standby daemon mds.foofs.smithi078.jdrsup assigned to filesystem foofs as rank 0 2024-09-17T09:49:40.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:40 smithi136 ceph-mon[49560]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-17T09:49:40.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:40 smithi136 ceph-mon[49560]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-17T09:49:40.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:40 smithi136 ceph-mon[49560]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-17T09:49:40.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:40 smithi136 ceph-mon[49560]: fsmap foofs:1/1 {0=foofs.smithi078.jdrsup=up:replay} 2024-09-17T09:49:41.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:41 smithi078 ceph-mon[63787]: pgmap v323: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 5.1 KiB/s wr, 2 op/s 2024-09-17T09:49:41.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:41 smithi136 ceph-mon[49560]: pgmap v323: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 5.1 KiB/s wr, 2 op/s 2024-09-17T09:49:42.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:42 smithi078 ceph-mon[63787]: reconnect by client.14548 172.21.15.78:0/3270440936 after 0 2024-09-17T09:49:42.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:42 smithi078 ceph-mon[63787]: reconnect by client.24321 172.21.15.136:0/3941243321 after 0 2024-09-17T09:49:42.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:42 smithi078 ceph-mon[63787]: mds.? [v2:172.21.15.78:6834/1849336238,v1:172.21.15.78:6835/1849336238] up:reconnect 2024-09-17T09:49:42.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:42 smithi078 ceph-mon[63787]: fsmap foofs:1/1 {0=foofs.smithi078.jdrsup=up:reconnect} 2024-09-17T09:49:42.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:42 smithi136 ceph-mon[49560]: reconnect by client.14548 172.21.15.78:0/3270440936 after 0 2024-09-17T09:49:42.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:42 smithi136 ceph-mon[49560]: reconnect by client.24321 172.21.15.136:0/3941243321 after 0 2024-09-17T09:49:42.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:42 smithi136 ceph-mon[49560]: mds.? [v2:172.21.15.78:6834/1849336238,v1:172.21.15.78:6835/1849336238] up:reconnect 2024-09-17T09:49:42.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:42 smithi136 ceph-mon[49560]: fsmap foofs:1/1 {0=foofs.smithi078.jdrsup=up:reconnect} 2024-09-17T09:49:43.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:43 smithi078 ceph-mon[63787]: mds.? [v2:172.21.15.78:6834/1849336238,v1:172.21.15.78:6835/1849336238] up:rejoin 2024-09-17T09:49:43.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:43 smithi078 ceph-mon[63787]: fsmap foofs:1/1 {0=foofs.smithi078.jdrsup=up:rejoin} 2024-09-17T09:49:43.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:43 smithi078 ceph-mon[63787]: daemon mds.foofs.smithi078.jdrsup is now active in filesystem foofs as rank 0 2024-09-17T09:49:43.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:43 smithi078 ceph-mon[63787]: pgmap v324: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 4 op/s 2024-09-17T09:49:43.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:43 smithi136 ceph-mon[49560]: mds.? [v2:172.21.15.78:6834/1849336238,v1:172.21.15.78:6835/1849336238] up:rejoin 2024-09-17T09:49:43.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:43 smithi136 ceph-mon[49560]: fsmap foofs:1/1 {0=foofs.smithi078.jdrsup=up:rejoin} 2024-09-17T09:49:43.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:43 smithi136 ceph-mon[49560]: daemon mds.foofs.smithi078.jdrsup is now active in filesystem foofs as rank 0 2024-09-17T09:49:43.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:43 smithi136 ceph-mon[49560]: pgmap v324: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 4 op/s 2024-09-17T09:49:44.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:44 smithi078 ceph-mon[63787]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-17T09:49:44.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:44 smithi078 ceph-mon[63787]: mds.? [v2:172.21.15.78:6834/1849336238,v1:172.21.15.78:6835/1849336238] up:active 2024-09-17T09:49:44.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:44 smithi078 ceph-mon[63787]: fsmap foofs:1 {0=foofs.smithi078.jdrsup=up:active} 2024-09-17T09:49:44.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:44 smithi136 ceph-mon[49560]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-17T09:49:44.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:44 smithi136 ceph-mon[49560]: mds.? [v2:172.21.15.78:6834/1849336238,v1:172.21.15.78:6835/1849336238] up:active 2024-09-17T09:49:44.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:44 smithi136 ceph-mon[49560]: fsmap foofs:1 {0=foofs.smithi078.jdrsup=up:active} 2024-09-17T09:49:45.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:45 smithi078 ceph-mon[63787]: pgmap v325: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 8 op/s 2024-09-17T09:49:45.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:45 smithi136 ceph-mon[49560]: pgmap v325: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 8 op/s 2024-09-17T09:49:47.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:47 smithi078 ceph-mon[63787]: pgmap v326: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 8 op/s 2024-09-17T09:49:47.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:47 smithi136 ceph-mon[49560]: pgmap v326: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 8 op/s 2024-09-17T09:49:49.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:49 smithi078 ceph-mon[63787]: pgmap v327: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 204 B/s wr, 11 op/s 2024-09-17T09:49:49.890 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-17T09:49:49.895 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:49 smithi136 ceph-mon[49560]: pgmap v327: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 204 B/s wr, 11 op/s 2024-09-17T09:49:50.576 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:49:50.582 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (10m) 18s ago 19m 18.2M - 0.27.0 11f11916f8cd 97db2373ce92 2024-09-17T09:49:50.582 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (6m) 18s ago 19m 6899k - 19.3.0-4963-ge646f729 f271290a0950 da528c5ca445 2024-09-17T09:49:50.582 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi136 smithi136 running (6m) 40s ago 18m 6899k - 19.3.0-4963-ge646f729 f271290a0950 909f405348cc 2024-09-17T09:49:50.582 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (10m) 18s ago 19m 53.9M - dad864ee21e9 db74b0f8e0a7 2024-09-17T09:49:50.582 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.jdrsup smithi078 running (23s) 18s ago 13m 11.7M - 19.3.0-4963-ge646f729 f271290a0950 809abd2f990d 2024-09-17T09:49:50.582 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi136.smslff smithi136 running (13m) 40s ago 13m 17.0M - 17.2.0 e1d6a67b021e 0c20e2b328d3 2024-09-17T09:49:50.582 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rsqofi smithi078 *:8443,9283,8765 running (9m) 18s ago 20m 522M - 19.3.0-4963-ge646f729 f271290a0950 7dda3108c9a0 2024-09-17T09:49:50.582 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi136.hicmlm smithi136 *:8443,9283,8765 running (8m) 40s ago 18m 456M - 19.3.0-4963-ge646f729 f271290a0950 fc518b58e07a 2024-09-17T09:49:50.582 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (8m) 18s ago 20m 65.5M 2048M 19.3.0-4963-ge646f729 f271290a0950 3ead101d3db1 2024-09-17T09:49:50.582 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi136 smithi136 running (7m) 40s ago 18m 46.7M 2048M 19.3.0-4963-ge646f729 f271290a0950 4fce88842b79 2024-09-17T09:49:50.582 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.tcwbvi smithi078 *:2049 running (12m) 18s ago 12m 275M - 4.0 e1d6a67b021e 30c87cb46555 2024-09-17T09:49:50.582 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi136.regjvc smithi136 *:2049 running (12m) 40s ago 12m 90.3M - 4.0 e1d6a67b021e b88e4dc24376 2024-09-17T09:49:50.582 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (9m) 18s ago 19m 10.4M - 1.7.0 72c9c2088986 52dad3adeeaa 2024-09-17T09:49:50.583 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi136 smithi136 *:9100 running (9m) 40s ago 18m 10.6M - 1.7.0 72c9c2088986 f2bc5856e35a 2024-09-17T09:49:50.583 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (6m) 18s ago 17m 433M 1936M 19.3.0-4963-ge646f729 f271290a0950 1ea81cc809b8 2024-09-17T09:49:50.583 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (5m) 18s ago 17m 399M 1936M 19.3.0-4963-ge646f729 f271290a0950 eda32560334b 2024-09-17T09:49:50.583 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (4m) 18s ago 16m 359M 1936M 19.3.0-4963-ge646f729 f271290a0950 83a32ad7c358 2024-09-17T09:49:50.583 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (3m) 18s ago 16m 341M 1936M 19.3.0-4963-ge646f729 f271290a0950 e2cd18998d2b 2024-09-17T09:49:50.583 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi136 running (2m) 40s ago 16m 354M 2704M 19.3.0-4963-ge646f729 f271290a0950 e2a18ad94664 2024-09-17T09:49:50.583 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi136 running (2m) 40s ago 15m 310M 2704M 19.3.0-4963-ge646f729 f271290a0950 df9b6954ce0f 2024-09-17T09:49:50.583 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi136 running (80s) 40s ago 15m 284M 2704M 19.3.0-4963-ge646f729 f271290a0950 c477e7043301 2024-09-17T09:49:50.583 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi136 running (45s) 40s ago 14m 11.2M 2704M 19.3.0-4963-ge646f729 f271290a0950 6c9da904d39c 2024-09-17T09:49:50.583 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (8m) 18s ago 18m 58.1M - 2.51.0 1d3b7f56885b 6e5e8c721eb0 2024-09-17T09:49:50.994 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:49:50.995 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-17T09:49:50.995 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:49:50.995 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:49:50.995 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-17T09:49:50.995 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:49:50.995 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:49:50.995 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-17T09:49:50.995 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 8 2024-09-17T09:49:50.995 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:49:50.995 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-17T09:49:50.995 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 1 2024-09-17T09:49:50.995 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:49:50.996 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-17T09:49:50.996 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 13 2024-09-17T09:49:50.996 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-17T09:49:50.996 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:49:51.350 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:49:51.350 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407", 2024-09-17T09:49:51.350 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-17T09:49:51.351 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T09:49:51.351 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-17T09:49:51.351 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-09-17T09:49:51.351 INFO:teuthology.orchestra.run.smithi078.stdout: "osd", 2024-09-17T09:49:51.351 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-17T09:49:51.351 INFO:teuthology.orchestra.run.smithi078.stdout: "mon" 2024-09-17T09:49:51.351 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-17T09:49:51.351 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "15/23 daemons upgraded", 2024-09-17T09:49:51.351 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading mds daemons", 2024-09-17T09:49:51.351 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-17T09:49:51.351 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:49:51.775 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-17T09:49:51.776 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:51 smithi078 ceph-mon[63787]: from='client.34484 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:49:51.776 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:51 smithi078 ceph-mon[63787]: from='client.34488 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:49:51.776 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:51 smithi078 ceph-mon[63787]: pgmap v328: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 193 B/s wr, 10 op/s 2024-09-17T09:49:51.776 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:51 smithi078 ceph-mon[63787]: from='client.34492 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:49:51.776 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:51 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/3970511790' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:51.776 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:51 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:51.776 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:51 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:51.776 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:51 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:49:51.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:51 smithi136 ceph-mon[49560]: from='client.34484 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:49:51.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:51 smithi136 ceph-mon[49560]: from='client.34488 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:49:51.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:51 smithi136 ceph-mon[49560]: pgmap v328: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 193 B/s wr, 10 op/s 2024-09-17T09:49:51.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:51 smithi136 ceph-mon[49560]: from='client.34492 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:49:51.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:51 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/3970511790' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:51.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:51 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:51.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:51 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:51.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:51 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:49:52.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:52 smithi136 ceph-mon[49560]: from='client.34502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:49:52.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:52 smithi136 ceph-mon[49560]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-17T09:49:52.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:52 smithi136 ceph-mon[49560]: Cluster is now healthy 2024-09-17T09:49:52.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:52 smithi136 ceph-mon[49560]: mds.? [v2:172.21.15.136:6832/4036583773,v1:172.21.15.136:6833/4036583773] up:boot 2024-09-17T09:49:52.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:52 smithi136 ceph-mon[49560]: fsmap foofs:1 {0=foofs.smithi078.jdrsup=up:active} 1 up:standby 2024-09-17T09:49:52.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:52 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mds metadata", "who": "foofs.smithi136.smslff"} : dispatch 2024-09-17T09:49:52.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:52 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/2351617480' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:49:52.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:52 smithi136 ceph-mon[49560]: pgmap v329: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 7.2 KiB/s wr, 12 op/s 2024-09-17T09:49:52.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:52 smithi078 ceph-mon[63787]: from='client.34502 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:49:52.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:52 smithi078 ceph-mon[63787]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-17T09:49:52.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:52 smithi078 ceph-mon[63787]: Cluster is now healthy 2024-09-17T09:49:52.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:52 smithi078 ceph-mon[63787]: mds.? [v2:172.21.15.136:6832/4036583773,v1:172.21.15.136:6833/4036583773] up:boot 2024-09-17T09:49:52.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:52 smithi078 ceph-mon[63787]: fsmap foofs:1 {0=foofs.smithi078.jdrsup=up:active} 1 up:standby 2024-09-17T09:49:52.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:52 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "mds metadata", "who": "foofs.smithi136.smslff"} : dispatch 2024-09-17T09:49:52.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:52 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/2351617480' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:49:52.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:52 smithi078 ceph-mon[63787]: pgmap v329: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 7.2 KiB/s wr, 12 op/s 2024-09-17T09:49:53.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:53 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:49:53.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:53 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:49:55.078 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:54 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:55.078 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:54 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:55.078 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:54 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:55.078 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:54 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:55.078 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:54 smithi136 ceph-mon[49560]: pgmap v330: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 7.2 KiB/s wr, 8 op/s 2024-09-17T09:49:55.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:54 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:55.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:54 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:55.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:54 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:55.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:54 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:55.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:54 smithi078 ceph-mon[63787]: pgmap v330: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 7.2 KiB/s wr, 8 op/s 2024-09-17T09:49:56.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:56 smithi136 ceph-mon[49560]: pgmap v331: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 343 KiB/s rd, 7.2 KiB/s wr, 5 op/s 2024-09-17T09:49:57.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:56 smithi078 ceph-mon[63787]: pgmap v331: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 343 KiB/s rd, 7.2 KiB/s wr, 5 op/s 2024-09-17T09:49:58.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: Detected new or changed devices on smithi136 2024-09-17T09:49:58.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:58.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:58.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:49:58.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:49:58.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:58.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:58.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:49:58.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:58.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:58.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:58.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:58.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:58.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: Upgrade: Setting container_image for all mds 2024-09-17T09:49:58.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:58.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi078.jdrsup"} : dispatch 2024-09-17T09:49:58.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi078.jdrsup"}]': finished 2024-09-17T09:49:58.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi136.smslff"} : dispatch 2024-09-17T09:49:58.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi136.smslff"}]': finished 2024-09-17T09:49:58.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:58.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: Upgrade: Setting container_image for all rgw 2024-09-17T09:49:58.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:58.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:58.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T09:49:58.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:58.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:58.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:58.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T09:49:58.850 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:58.850 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:58.850 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: Upgrade: Setting container_image for all iscsi 2024-09-17T09:49:58.850 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:58.850 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:49:58 smithi078 ceph-mon[63787]: pgmap v332: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.2 KiB/s rd, 7.2 KiB/s wr, 5 op/s 2024-09-17T09:49:59.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: Detected new or changed devices on smithi136 2024-09-17T09:49:59.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:59.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:59.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:49:59.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:49:59.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:59.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:59.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:49:59.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:59.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:59.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:59.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:59.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:59.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: Upgrade: Setting container_image for all mds 2024-09-17T09:49:59.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:59.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi078.jdrsup"} : dispatch 2024-09-17T09:49:59.228 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi078.jdrsup"}]': finished 2024-09-17T09:49:59.228 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi136.smslff"} : dispatch 2024-09-17T09:49:59.228 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi136.smslff"}]': finished 2024-09-17T09:49:59.228 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:59.228 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: Upgrade: Setting container_image for all rgw 2024-09-17T09:49:59.228 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:59.228 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:59.228 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T09:49:59.228 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:59.228 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:59.228 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:59.228 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T09:49:59.228 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:59.229 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:49:59.229 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: Upgrade: Setting container_image for all iscsi 2024-09-17T09:49:59.229 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:49:59.229 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:49:58 smithi136 ceph-mon[49560]: pgmap v332: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.2 KiB/s rd, 7.2 KiB/s wr, 5 op/s 2024-09-17T09:50:01.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:00 smithi078 ceph-mon[63787]: Upgrade: Updating nfs.foo.0.0.smithi078.tcwbvi (1/2) 2024-09-17T09:50:01.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:00 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:01.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:00 smithi078 ceph-mon[63787]: Creating key for client.nfs.foo.0.0.smithi078.tcwbvi 2024-09-17T09:50:01.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:00 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.tcwbvi", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-17T09:50:01.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:00 smithi078 ceph-mon[63787]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-17T09:50:01.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:00 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-17T09:50:01.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:00 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T09:50:01.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:00 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:50:01.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:00 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-17T09:50:01.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:00 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T09:50:01.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:00 smithi078 ceph-mon[63787]: Rados config object exists: conf-nfs.foo 2024-09-17T09:50:01.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:00 smithi078 ceph-mon[63787]: Creating key for client.nfs.foo.0.0.smithi078.tcwbvi-rgw 2024-09-17T09:50:01.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:00 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.tcwbvi-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-17T09:50:01.098 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:00 smithi078 ceph-mon[63787]: Bind address in nfs.foo.0.0.smithi078.tcwbvi's ganesha conf is defaulting to empty 2024-09-17T09:50:01.098 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:00 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:50:01.098 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:00 smithi078 ceph-mon[63787]: Deploying daemon nfs.foo.0.0.smithi078.tcwbvi on smithi078 2024-09-17T09:50:01.098 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:00 smithi078 ceph-mon[63787]: overall HEALTH_OK 2024-09-17T09:50:01.098 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:00 smithi078 ceph-mon[63787]: pgmap v333: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 7.1 KiB/s wr, 3 op/s 2024-09-17T09:50:01.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:00 smithi136 ceph-mon[49560]: Upgrade: Updating nfs.foo.0.0.smithi078.tcwbvi (1/2) 2024-09-17T09:50:01.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:00 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:01.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:00 smithi136 ceph-mon[49560]: Creating key for client.nfs.foo.0.0.smithi078.tcwbvi 2024-09-17T09:50:01.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:00 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.tcwbvi", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-17T09:50:01.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:00 smithi136 ceph-mon[49560]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-17T09:50:01.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:00 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-17T09:50:01.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:00 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T09:50:01.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:00 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:50:01.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:00 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-17T09:50:01.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:00 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T09:50:01.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:00 smithi136 ceph-mon[49560]: Rados config object exists: conf-nfs.foo 2024-09-17T09:50:01.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:00 smithi136 ceph-mon[49560]: Creating key for client.nfs.foo.0.0.smithi078.tcwbvi-rgw 2024-09-17T09:50:01.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:00 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.tcwbvi-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-17T09:50:01.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:00 smithi136 ceph-mon[49560]: Bind address in nfs.foo.0.0.smithi078.tcwbvi's ganesha conf is defaulting to empty 2024-09-17T09:50:01.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:00 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:50:01.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:00 smithi136 ceph-mon[49560]: Deploying daemon nfs.foo.0.0.smithi078.tcwbvi on smithi078 2024-09-17T09:50:01.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:00 smithi136 ceph-mon[49560]: overall HEALTH_OK 2024-09-17T09:50:01.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:00 smithi136 ceph-mon[49560]: pgmap v333: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 7.1 KiB/s wr, 3 op/s 2024-09-17T09:50:02.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:02 smithi136 ceph-mon[49560]: pgmap v334: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 7.3 KiB/s wr, 3 op/s 2024-09-17T09:50:03.034 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:02 smithi078 ceph-mon[63787]: pgmap v334: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 7.3 KiB/s wr, 3 op/s 2024-09-17T09:50:03.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:03 smithi136 ceph-mon[49560]: from='mds.? [v2:172.21.15.78:6834/1849336238,v1:172.21.15.78:6835/1849336238]' entity='mds.foofs.smithi078.jdrsup' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.78:0/3270440936"} : dispatch 2024-09-17T09:50:04.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:03 smithi078 ceph-mon[63787]: from='mds.? [v2:172.21.15.78:6834/1849336238,v1:172.21.15.78:6835/1849336238]' entity='mds.foofs.smithi078.jdrsup' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.78:0/3270440936"} : dispatch 2024-09-17T09:50:04.959 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:04 smithi136 ceph-mon[49560]: Evicting (and blocklisting) client session 14548 (172.21.15.78:0/3270440936) 2024-09-17T09:50:04.959 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:04 smithi136 ceph-mon[49560]: from='mds.? [v2:172.21.15.78:6834/1849336238,v1:172.21.15.78:6835/1849336238]' entity='mds.foofs.smithi078.jdrsup' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.78:0/3270440936"}]': finished 2024-09-17T09:50:04.959 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:04 smithi136 ceph-mon[49560]: osdmap e160: 8 total, 8 up, 8 in 2024-09-17T09:50:04.959 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:04 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:04.959 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:04 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:04.959 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:04 smithi136 ceph-mon[49560]: pgmap v336: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s rd, 613 B/s wr, 1 op/s 2024-09-17T09:50:05.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:04 smithi078 ceph-mon[63787]: Evicting (and blocklisting) client session 14548 (172.21.15.78:0/3270440936) 2024-09-17T09:50:05.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:04 smithi078 ceph-mon[63787]: from='mds.? [v2:172.21.15.78:6834/1849336238,v1:172.21.15.78:6835/1849336238]' entity='mds.foofs.smithi078.jdrsup' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.78:0/3270440936"}]': finished 2024-09-17T09:50:05.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:04 smithi078 ceph-mon[63787]: osdmap e160: 8 total, 8 up, 8 in 2024-09-17T09:50:05.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:04 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:05.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:04 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:05.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:04 smithi078 ceph-mon[63787]: pgmap v336: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s rd, 613 B/s wr, 1 op/s 2024-09-17T09:50:06.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:05 smithi136 ceph-mon[49560]: Upgrade: Updating nfs.foo.1.0.smithi136.regjvc (2/2) 2024-09-17T09:50:06.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:05 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:06.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:05 smithi136 ceph-mon[49560]: Creating key for client.nfs.foo.1.0.smithi136.regjvc 2024-09-17T09:50:06.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:05 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi136.regjvc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-17T09:50:06.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:05 smithi136 ceph-mon[49560]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-17T09:50:06.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:05 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-17T09:50:06.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:05 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T09:50:06.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:05 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:50:06.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:05 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-17T09:50:06.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:05 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T09:50:06.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:05 smithi136 ceph-mon[49560]: Rados config object exists: conf-nfs.foo 2024-09-17T09:50:06.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:05 smithi136 ceph-mon[49560]: Creating key for client.nfs.foo.1.0.smithi136.regjvc-rgw 2024-09-17T09:50:06.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:05 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi136.regjvc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-17T09:50:06.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:05 smithi136 ceph-mon[49560]: Bind address in nfs.foo.1.0.smithi136.regjvc's ganesha conf is defaulting to empty 2024-09-17T09:50:06.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:05 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:50:06.227 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:05 smithi136 ceph-mon[49560]: Deploying daemon nfs.foo.1.0.smithi136.regjvc on smithi136 2024-09-17T09:50:06.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:05 smithi078 ceph-mon[63787]: Upgrade: Updating nfs.foo.1.0.smithi136.regjvc (2/2) 2024-09-17T09:50:06.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:05 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:06.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:05 smithi078 ceph-mon[63787]: Creating key for client.nfs.foo.1.0.smithi136.regjvc 2024-09-17T09:50:06.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:05 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi136.regjvc", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-17T09:50:06.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:05 smithi078 ceph-mon[63787]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-17T09:50:06.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:05 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-17T09:50:06.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:05 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T09:50:06.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:05 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:50:06.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:05 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-17T09:50:06.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:05 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T09:50:06.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:05 smithi078 ceph-mon[63787]: Rados config object exists: conf-nfs.foo 2024-09-17T09:50:06.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:05 smithi078 ceph-mon[63787]: Creating key for client.nfs.foo.1.0.smithi136.regjvc-rgw 2024-09-17T09:50:06.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:05 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi136.regjvc-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-17T09:50:06.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:05 smithi078 ceph-mon[63787]: Bind address in nfs.foo.1.0.smithi136.regjvc's ganesha conf is defaulting to empty 2024-09-17T09:50:06.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:05 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:50:06.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:05 smithi078 ceph-mon[63787]: Deploying daemon nfs.foo.1.0.smithi136.regjvc on smithi136 2024-09-17T09:50:07.212 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:06 smithi136 ceph-mon[49560]: pgmap v337: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s rd, 716 B/s wr, 1 op/s 2024-09-17T09:50:07.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:06 smithi078 ceph-mon[63787]: pgmap v337: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s rd, 716 B/s wr, 1 op/s 2024-09-17T09:50:08.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:07 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:50:08.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:07 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:50:09.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:09 smithi078 ceph-mon[63787]: pgmap v338: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.3 KiB/s rd, 1.5 KiB/s wr, 5 op/s 2024-09-17T09:50:09.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:09 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:09.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:09 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:09.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:09 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:50:09.431 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:09 smithi136 ceph-mon[49560]: pgmap v338: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.3 KiB/s rd, 1.5 KiB/s wr, 5 op/s 2024-09-17T09:50:09.431 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:09 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:09.432 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:09 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:09.432 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:09 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:50:10.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:10 smithi136 ceph-mon[49560]: pgmap v339: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.3 KiB/s rd, 1.5 KiB/s wr, 5 op/s 2024-09-17T09:50:11.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:10 smithi078 ceph-mon[63787]: pgmap v339: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.3 KiB/s rd, 1.5 KiB/s wr, 5 op/s 2024-09-17T09:50:12.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:12 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:12.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:12 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:12.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:12 smithi136 ceph-mon[49560]: Evicting (and blocklisting) client session 24321 (172.21.15.136:0/3941243321) 2024-09-17T09:50:12.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:12 smithi136 ceph-mon[49560]: from='mds.? [v2:172.21.15.78:6834/1849336238,v1:172.21.15.78:6835/1849336238]' entity='mds.foofs.smithi078.jdrsup' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.136:0/3941243321"} : dispatch 2024-09-17T09:50:12.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:12 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:12.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:12 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:12.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:12 smithi136 ceph-mon[49560]: pgmap v340: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.4 KiB/s rd, 1.4 KiB/s wr, 5 op/s 2024-09-17T09:50:13.005 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:12 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:13.005 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:12 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:13.005 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:12 smithi078 ceph-mon[63787]: Evicting (and blocklisting) client session 24321 (172.21.15.136:0/3941243321) 2024-09-17T09:50:13.005 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:12 smithi078 ceph-mon[63787]: from='mds.? [v2:172.21.15.78:6834/1849336238,v1:172.21.15.78:6835/1849336238]' entity='mds.foofs.smithi078.jdrsup' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.136:0/3941243321"} : dispatch 2024-09-17T09:50:13.005 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:12 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:13.005 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:12 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:13.005 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:12 smithi078 ceph-mon[63787]: pgmap v340: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.4 KiB/s rd, 1.4 KiB/s wr, 5 op/s 2024-09-17T09:50:13.938 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:13 smithi136 ceph-mon[49560]: from='mds.? [v2:172.21.15.78:6834/1849336238,v1:172.21.15.78:6835/1849336238]' entity='mds.foofs.smithi078.jdrsup' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.136:0/3941243321"}]': finished 2024-09-17T09:50:13.939 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:13 smithi136 ceph-mon[49560]: osdmap e161: 8 total, 8 up, 8 in 2024-09-17T09:50:13.939 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:13 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:13.939 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:13 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:14.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:13 smithi078 ceph-mon[63787]: from='mds.? [v2:172.21.15.78:6834/1849336238,v1:172.21.15.78:6835/1849336238]' entity='mds.foofs.smithi078.jdrsup' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.136:0/3941243321"}]': finished 2024-09-17T09:50:14.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:13 smithi078 ceph-mon[63787]: osdmap e161: 8 total, 8 up, 8 in 2024-09-17T09:50:14.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:13 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:14.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:13 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:15.346 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:15.346 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:15 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:15.346 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:15 smithi136 ceph-mon[49560]: pgmap v342: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.2 KiB/s rd, 1.7 KiB/s wr, 6 op/s 2024-09-17T09:50:15.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:15.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:15 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:15.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:15 smithi078 ceph-mon[63787]: pgmap v342: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.2 KiB/s rd, 1.7 KiB/s wr, 6 op/s 2024-09-17T09:50:16.635 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:16 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:16.635 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:16 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:16.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:16 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:16.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:16 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:17.593 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:17 smithi078 ceph-mon[63787]: pgmap v343: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.2 KiB/s rd, 1.6 KiB/s wr, 6 op/s 2024-09-17T09:50:17.688 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:17 smithi136 ceph-mon[49560]: pgmap v343: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 4.2 KiB/s rd, 1.6 KiB/s wr, 6 op/s 2024-09-17T09:50:18.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:18.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:18.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:50:18.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:50:18.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:18.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:18.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:50:18.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:18.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:18.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:18.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:18.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:18.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:18.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:18.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:18.977 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:18.978 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:18.978 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:18.978 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:18 smithi136 ceph-mon[49560]: Upgrade: Setting container_image for all nfs 2024-09-17T09:50:18.978 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:18.978 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi078.tcwbvi"} : dispatch 2024-09-17T09:50:18.978 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi078.tcwbvi"}]': finished 2024-09-17T09:50:18.978 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi136.regjvc"} : dispatch 2024-09-17T09:50:18.978 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi136.regjvc"}]': finished 2024-09-17T09:50:18.978 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:18.978 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:18 smithi136 ceph-mon[49560]: Upgrade: Setting container_image for all nvmeof 2024-09-17T09:50:18.978 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:18.978 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:18.978 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:18.979 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:18 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:18.979 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:18 smithi136 ceph-mon[49560]: pgmap v344: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.3 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-09-17T09:50:19.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:19.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:19.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:50:19.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:50:19.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:19.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:19.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:50:19.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:19.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:19.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:19.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:19.098 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:19.098 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:19.098 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:19.098 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:19.098 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:19.098 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:19.098 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:19.098 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:18 smithi078 ceph-mon[63787]: Upgrade: Setting container_image for all nfs 2024-09-17T09:50:19.098 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:19.098 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi078.tcwbvi"} : dispatch 2024-09-17T09:50:19.098 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi078.tcwbvi"}]': finished 2024-09-17T09:50:19.098 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi136.regjvc"} : dispatch 2024-09-17T09:50:19.098 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi136.regjvc"}]': finished 2024-09-17T09:50:19.099 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:19.099 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:18 smithi078 ceph-mon[63787]: Upgrade: Setting container_image for all nvmeof 2024-09-17T09:50:19.099 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:19.099 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:19.099 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:19.099 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:18 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:19.099 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:18 smithi078 ceph-mon[63787]: pgmap v344: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.3 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-09-17T09:50:20.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:19 smithi078 ceph-mon[63787]: Upgrade: Updating grafana.smithi078 2024-09-17T09:50:20.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:19 smithi136 ceph-mon[49560]: Upgrade: Updating grafana.smithi078 2024-09-17T09:50:21.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:20 smithi078 ceph-mon[63787]: Deploying daemon grafana.smithi078 on smithi078 2024-09-17T09:50:21.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:20 smithi078 ceph-mon[63787]: pgmap v345: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.3 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-09-17T09:50:21.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:20 smithi136 ceph-mon[49560]: Deploying daemon grafana.smithi078 on smithi078 2024-09-17T09:50:21.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:20 smithi136 ceph-mon[49560]: pgmap v345: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.3 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-09-17T09:50:22.345 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-17T09:50:22.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:22 smithi136 ceph-mon[49560]: from='client.34554 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:50:22.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:22 smithi136 ceph-mon[49560]: pgmap v346: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.2 KiB/s rd, 1023 B/s wr, 3 op/s 2024-09-17T09:50:23.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:22 smithi078 ceph-mon[63787]: from='client.34554 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:50:23.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:22 smithi078 ceph-mon[63787]: pgmap v346: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.2 KiB/s rd, 1023 B/s wr, 3 op/s 2024-09-17T09:50:23.311 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:50:23.311 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (10m) 9s ago 19m 18.2M - 0.27.0 11f11916f8cd 97db2373ce92 2024-09-17T09:50:23.311 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (7m) 9s ago 19m 6899k - 19.3.0-4963-ge646f729 f271290a0950 da528c5ca445 2024-09-17T09:50:23.311 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi136 smithi136 running (7m) 11s ago 19m 6899k - 19.3.0-4963-ge646f729 f271290a0950 909f405348cc 2024-09-17T09:50:23.312 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (10m) 9s ago 19m 53.0M - dad864ee21e9 db74b0f8e0a7 2024-09-17T09:50:23.312 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.jdrsup smithi078 running (56s) 9s ago 13m 34.3M - 19.3.0-4963-ge646f729 f271290a0950 809abd2f990d 2024-09-17T09:50:23.312 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi136.smslff smithi136 running (32s) 11s ago 13m 15.5M - 19.3.0-4963-ge646f729 f271290a0950 516254567aa2 2024-09-17T09:50:23.312 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rsqofi smithi078 *:8443,9283,8765 running (9m) 9s ago 21m 524M - 19.3.0-4963-ge646f729 f271290a0950 7dda3108c9a0 2024-09-17T09:50:23.312 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi136.hicmlm smithi136 *:8443,9283,8765 running (9m) 11s ago 19m 457M - 19.3.0-4963-ge646f729 f271290a0950 fc518b58e07a 2024-09-17T09:50:23.312 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (8m) 9s ago 21m 64.2M 2048M 19.3.0-4963-ge646f729 f271290a0950 3ead101d3db1 2024-09-17T09:50:23.312 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi136 smithi136 running (8m) 11s ago 19m 49.6M 2048M 19.3.0-4963-ge646f729 f271290a0950 4fce88842b79 2024-09-17T09:50:23.312 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.tcwbvi smithi078 *:2049 running (20s) 9s ago 13m 66.5M - 5.9 f271290a0950 d399aeec32cb 2024-09-17T09:50:23.312 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi136.regjvc smithi136 *:2049 running (15s) 11s ago 13m 19.8M - 5.9 f271290a0950 65fc579b5d9f 2024-09-17T09:50:23.312 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (10m) 9s ago 19m 10.7M - 1.7.0 72c9c2088986 52dad3adeeaa 2024-09-17T09:50:23.312 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi136 smithi136 *:9100 running (10m) 11s ago 18m 12.5M - 1.7.0 72c9c2088986 f2bc5856e35a 2024-09-17T09:50:23.312 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (6m) 9s ago 18m 434M 1936M 19.3.0-4963-ge646f729 f271290a0950 1ea81cc809b8 2024-09-17T09:50:23.313 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (5m) 9s ago 17m 401M 1936M 19.3.0-4963-ge646f729 f271290a0950 eda32560334b 2024-09-17T09:50:23.313 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (4m) 9s ago 17m 362M 1936M 19.3.0-4963-ge646f729 f271290a0950 83a32ad7c358 2024-09-17T09:50:23.313 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (4m) 9s ago 16m 344M 1936M 19.3.0-4963-ge646f729 f271290a0950 e2cd18998d2b 2024-09-17T09:50:23.313 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi136 running (3m) 11s ago 16m 371M 2704M 19.3.0-4963-ge646f729 f271290a0950 e2a18ad94664 2024-09-17T09:50:23.313 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi136 running (2m) 11s ago 16m 335M 2704M 19.3.0-4963-ge646f729 f271290a0950 df9b6954ce0f 2024-09-17T09:50:23.313 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi136 running (113s) 11s ago 15m 295M 2704M 19.3.0-4963-ge646f729 f271290a0950 c477e7043301 2024-09-17T09:50:23.313 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi136 running (77s) 11s ago 15m 290M 2704M 19.3.0-4963-ge646f729 f271290a0950 6c9da904d39c 2024-09-17T09:50:23.313 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (9m) 9s ago 18m 58.2M - 2.51.0 1d3b7f56885b 6e5e8c721eb0 2024-09-17T09:50:23.740 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:50:23.740 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-17T09:50:23.740 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:50:23.741 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:50:23.741 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-17T09:50:23.741 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:50:23.741 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:50:23.741 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-17T09:50:23.741 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 8 2024-09-17T09:50:23.741 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:50:23.741 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-17T09:50:23.741 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:50:23.741 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:50:23.741 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-17T09:50:23.741 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 14 2024-09-17T09:50:23.742 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-17T09:50:23.742 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:50:23.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:23 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:50:23.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:23 smithi136 ceph-mon[49560]: from='client.34558 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:50:23.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:23 smithi136 ceph-mon[49560]: from='client.34562 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:50:24.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:23 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:50:24.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:23 smithi078 ceph-mon[63787]: from='client.34558 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:50:24.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:23 smithi078 ceph-mon[63787]: from='client.34562 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:50:24.135 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:50:24.135 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407", 2024-09-17T09:50:24.135 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-17T09:50:24.136 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T09:50:24.136 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-17T09:50:24.136 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-09-17T09:50:24.136 INFO:teuthology.orchestra.run.smithi078.stdout: "nfs", 2024-09-17T09:50:24.136 INFO:teuthology.orchestra.run.smithi078.stdout: "osd", 2024-09-17T09:50:24.136 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-09-17T09:50:24.136 INFO:teuthology.orchestra.run.smithi078.stdout: "mds", 2024-09-17T09:50:24.136 INFO:teuthology.orchestra.run.smithi078.stdout: "crash" 2024-09-17T09:50:24.136 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-17T09:50:24.136 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "18/23 daemons upgraded", 2024-09-17T09:50:24.136 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading grafana daemons", 2024-09-17T09:50:24.136 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-17T09:50:24.136 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:50:24.579 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-17T09:50:24.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:24 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/2170158335' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:24.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:24 smithi078 ceph-mon[63787]: from='client.34570 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:50:24.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:24 smithi078 ceph-mon[63787]: pgmap v347: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 173 B/s rd, 346 B/s wr, 0 op/s 2024-09-17T09:50:24.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:24 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/3816011088' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:50:24.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:24 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/2170158335' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:24.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:24 smithi136 ceph-mon[49560]: from='client.34570 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:50:24.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:24 smithi136 ceph-mon[49560]: pgmap v347: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 173 B/s rd, 346 B/s wr, 0 op/s 2024-09-17T09:50:24.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:24 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/3816011088' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:50:26.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:26 smithi136 ceph-mon[49560]: pgmap v348: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-17T09:50:27.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:26 smithi078 ceph-mon[63787]: pgmap v348: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-17T09:50:28.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:28 smithi078 ceph-mon[63787]: pgmap v349: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 341 B/s wr, 0 op/s 2024-09-17T09:50:28.908 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:28 smithi136 ceph-mon[49560]: pgmap v349: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 341 B/s wr, 0 op/s 2024-09-17T09:50:30.736 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:30 smithi078 ceph-mon[63787]: pgmap v350: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:50:30.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:30 smithi136 ceph-mon[49560]: pgmap v350: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:50:33.241 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:33 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:33.242 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:33 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:33.242 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:33 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:50:33.242 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:33 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:33.242 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:33 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:33.242 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:33 smithi078 ceph-mon[63787]: pgmap v351: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:50:33.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:33 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:33.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:33 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:33.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:33 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:50:33.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:33 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:33.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:33 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:33.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:33 smithi136 ceph-mon[49560]: pgmap v351: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:50:34.915 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:34 smithi078 ceph-mon[63787]: pgmap v352: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:50:34.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:34 smithi136 ceph-mon[49560]: pgmap v352: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:50:36.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:36 smithi136 ceph-mon[49560]: pgmap v353: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:50:37.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:36 smithi078 ceph-mon[63787]: pgmap v353: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:50:39.044 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:50:39.044 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:39.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:39.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:39.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:38 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:39.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:38 smithi078 ceph-mon[63787]: pgmap v354: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:50:39.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:50:39.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:39.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:39.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:39.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:38 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:39.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:38 smithi136 ceph-mon[49560]: pgmap v354: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:50:40.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:40 smithi136 ceph-mon[49560]: pgmap v355: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:50:41.026 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:40 smithi078 ceph-mon[63787]: pgmap v355: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:50:42.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:42 smithi136 ceph-mon[49560]: pgmap v356: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:50:43.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:42 smithi078 ceph-mon[63787]: pgmap v356: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:50:44.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:44 smithi136 ceph-mon[49560]: pgmap v357: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:50:45.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:44 smithi078 ceph-mon[63787]: pgmap v357: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:50:46.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:46.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:46.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:50:46.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:50:46.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:46.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:46.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-17T09:50:46.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T09:50:46.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:50:46.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.727 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.728 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.728 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.728 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.728 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.728 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.728 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:46.728 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-17T09:50:46.728 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T09:50:46.728 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T09:50:46.728 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T09:50:46.728 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-17T09:50:46.728 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T09:50:46.728 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-17T09:50:46.729 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-17T09:50:46.729 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-17T09:50:46.729 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T09:50:46.729 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-17T09:50:46.729 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-17T09:50:46.729 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-17T09:50:46.729 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T09:50:46.729 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T09:50:46.729 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-17T09:50:46.729 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T09:50:46.729 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-17T09:50:46.729 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-17T09:50:46.729 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-17T09:50:46.730 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T09:50:46.730 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-17T09:50:46.730 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T09:50:46.730 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T09:50:46.730 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T09:50:46.730 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T09:50:46.730 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T09:50:46.730 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T09:50:46.730 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T09:50:46.730 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-17T09:50:46.730 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T09:50:46.730 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:50:46.730 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:50:46.730 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:50:46.731 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:46.731 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:46.731 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:50:46.731 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:50:46.731 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:50:46.731 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:46.731 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:46.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:46.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:46.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:50:46.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:50:46.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:46.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:46.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-17T09:50:46.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T09:50:46.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:50:46.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:50:46.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:46.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-17T09:50:46.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T09:50:46.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T09:50:46.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T09:50:46.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-17T09:50:46.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T09:50:46.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-17T09:50:46.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-17T09:50:46.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-17T09:50:46.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T09:50:46.850 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-17T09:50:46.850 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-17T09:50:46.850 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-17T09:50:46.850 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T09:50:46.850 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T09:50:46.850 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-17T09:50:46.850 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T09:50:46.850 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-17T09:50:46.850 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-17T09:50:46.850 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-17T09:50:46.851 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T09:50:46.851 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-17T09:50:46.851 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T09:50:46.851 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T09:50:46.851 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T09:50:46.851 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T09:50:46.851 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T09:50:46.851 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T09:50:46.851 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T09:50:46.851 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-17T09:50:46.851 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T09:50:46.851 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:50:46.851 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:50:46.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:50:46.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:46.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:46.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:50:46.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:50:46.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:50:46.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:46.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:47.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:47 smithi078 ceph-mon[63787]: Upgrade: Finalizing container_image settings 2024-09-17T09:50:47.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:47 smithi078 ceph-mon[63787]: Upgrade: Complete! 2024-09-17T09:50:47.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:47 smithi078 ceph-mon[63787]: pgmap v358: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:50:47.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:47 smithi136 ceph-mon[49560]: Upgrade: Finalizing container_image settings 2024-09-17T09:50:47.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:47 smithi136 ceph-mon[49560]: Upgrade: Complete! 2024-09-17T09:50:47.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:47 smithi136 ceph-mon[49560]: pgmap v358: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:50:49.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:48 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:49.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:48 smithi136 ceph-mon[49560]: pgmap v359: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:50:49.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:48 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:50:49.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:48 smithi078 ceph-mon[63787]: pgmap v359: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:50:50.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:50 smithi136 ceph-mon[49560]: pgmap v360: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:50:51.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:50 smithi078 ceph-mon[63787]: pgmap v360: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:50:52.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:52 smithi136 ceph-mon[49560]: pgmap v361: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:50:53.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:52 smithi078 ceph-mon[63787]: pgmap v361: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:50:53.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:53 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:50:54.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:53 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:50:54.947 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:54 smithi078 ceph-mon[63787]: pgmap v362: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:50:54.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:54 smithi136 ceph-mon[49560]: pgmap v362: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:50:55.575 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -e sha1=e646f72961223a04d41eb40dad875d1628d78516 -- bash -c 'ceph orch ps' 2024-09-17T09:50:55.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:55 smithi078 ceph-mon[63787]: from='client.34578 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:50:55.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:55 smithi136 ceph-mon[49560]: from='client.34578 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:50:56.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:56 smithi078 ceph-mon[63787]: pgmap v363: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:50:56.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:56 smithi136 ceph-mon[49560]: pgmap v363: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:50:57.656 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:50:57.656 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (11m) 19s ago 20m 18.2M - 0.27.0 11f11916f8cd 97db2373ce92 2024-09-17T09:50:57.656 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (7m) 19s ago 20m 6899k - 19.3.0-4963-ge646f729 f271290a0950 da528c5ca445 2024-09-17T09:50:57.656 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi136 smithi136 running (7m) 45s ago 19m 6899k - 19.3.0-4963-ge646f729 f271290a0950 909f405348cc 2024-09-17T09:50:57.656 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (25s) 19s ago 20m 44.6M - 10.4.0 c8b91775d855 bb0a24f1977c 2024-09-17T09:50:57.656 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.jdrsup smithi078 running (90s) 19s ago 14m 34.4M - 19.3.0-4963-ge646f729 f271290a0950 809abd2f990d 2024-09-17T09:50:57.656 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi136.smslff smithi136 running (67s) 45s ago 14m 15.5M - 19.3.0-4963-ge646f729 f271290a0950 516254567aa2 2024-09-17T09:50:57.656 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rsqofi smithi078 *:8443,9283,8765 running (10m) 19s ago 21m 533M - 19.3.0-4963-ge646f729 f271290a0950 7dda3108c9a0 2024-09-17T09:50:57.656 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi136.hicmlm smithi136 *:8443,9283,8765 running (9m) 45s ago 19m 457M - 19.3.0-4963-ge646f729 f271290a0950 fc518b58e07a 2024-09-17T09:50:57.656 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (9m) 19s ago 21m 67.9M 2048M 19.3.0-4963-ge646f729 f271290a0950 3ead101d3db1 2024-09-17T09:50:57.656 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi136 smithi136 running (8m) 45s ago 19m 49.6M 2048M 19.3.0-4963-ge646f729 f271290a0950 4fce88842b79 2024-09-17T09:50:57.656 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.tcwbvi smithi078 *:2049 running (54s) 19s ago 14m 67.3M - 5.9 f271290a0950 d399aeec32cb 2024-09-17T09:50:57.657 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi136.regjvc smithi136 *:2049 running (49s) 45s ago 13m 19.8M - 5.9 f271290a0950 65fc579b5d9f 2024-09-17T09:50:57.657 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (11m) 19s ago 20m 10.9M - 1.7.0 72c9c2088986 52dad3adeeaa 2024-09-17T09:50:57.657 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi136 smithi136 *:9100 running (10m) 45s ago 19m 12.5M - 1.7.0 72c9c2088986 f2bc5856e35a 2024-09-17T09:50:57.657 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (7m) 19s ago 18m 434M 1936M 19.3.0-4963-ge646f729 f271290a0950 1ea81cc809b8 2024-09-17T09:50:57.657 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (6m) 19s ago 18m 402M 1936M 19.3.0-4963-ge646f729 f271290a0950 eda32560334b 2024-09-17T09:50:57.657 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (5m) 19s ago 17m 362M 1936M 19.3.0-4963-ge646f729 f271290a0950 83a32ad7c358 2024-09-17T09:50:57.657 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (4m) 19s ago 17m 344M 1936M 19.3.0-4963-ge646f729 f271290a0950 e2cd18998d2b 2024-09-17T09:50:57.657 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi136 running (3m) 45s ago 17m 371M 2704M 19.3.0-4963-ge646f729 f271290a0950 e2a18ad94664 2024-09-17T09:50:57.657 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi136 running (3m) 45s ago 16m 335M 2704M 19.3.0-4963-ge646f729 f271290a0950 df9b6954ce0f 2024-09-17T09:50:57.657 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi136 running (2m) 45s ago 16m 295M 2704M 19.3.0-4963-ge646f729 f271290a0950 c477e7043301 2024-09-17T09:50:57.657 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi136 running (112s) 45s ago 15m 290M 2704M 19.3.0-4963-ge646f729 f271290a0950 6c9da904d39c 2024-09-17T09:50:57.657 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (9m) 19s ago 19m 58.2M - 2.51.0 1d3b7f56885b 6e5e8c721eb0 2024-09-17T09:50:58.573 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -e sha1=e646f72961223a04d41eb40dad875d1628d78516 -- bash -c 'ceph versions' 2024-09-17T09:50:58.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:58 smithi078 ceph-mon[63787]: from='client.34582 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:50:58.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:50:58 smithi078 ceph-mon[63787]: pgmap v364: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:50:58.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:58 smithi136 ceph-mon[49560]: from='client.34582 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:50:58.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:50:58 smithi136 ceph-mon[49560]: pgmap v364: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:51:00.703 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:51:00.703 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-17T09:51:00.703 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:51:00.703 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:51:00.703 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-17T09:51:00.703 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:51:00.703 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:51:00.703 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-17T09:51:00.703 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 8 2024-09-17T09:51:00.703 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:51:00.704 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-17T09:51:00.704 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:51:00.704 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:51:00.704 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-17T09:51:00.704 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 14 2024-09-17T09:51:00.704 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-17T09:51:00.704 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:51:00.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:00 smithi136 ceph-mon[49560]: pgmap v365: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:01.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:00 smithi078 ceph-mon[63787]: pgmap v365: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:01.495 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -e sha1=e646f72961223a04d41eb40dad875d1628d78516 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-09-17T09:51:01.793 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:01 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/2703035326' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:51:01.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:01 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/2703035326' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:51:02.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:02 smithi078 ceph-mon[63787]: pgmap v366: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:02.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:02 smithi136 ceph-mon[49560]: pgmap v366: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:03.473 INFO:teuthology.orchestra.run.smithi078.stdout:wait for servicemap items w/ changing names to refresh 2024-09-17T09:51:04.129 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -e sha1=e646f72961223a04d41eb40dad875d1628d78516 -- bash -c 'sleep 60' 2024-09-17T09:51:04.704 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:04 smithi078 ceph-mon[63787]: pgmap v367: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:51:04.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:04 smithi136 ceph-mon[49560]: pgmap v367: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:51:06.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:06 smithi136 ceph-mon[49560]: pgmap v368: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:07.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:06 smithi078 ceph-mon[63787]: pgmap v368: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:08.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:08 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:51:08.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:08 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:51:09.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:09 smithi078 ceph-mon[63787]: pgmap v369: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:51:09.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:09 smithi136 ceph-mon[49560]: pgmap v369: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:51:10.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:10 smithi136 ceph-mon[49560]: pgmap v370: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:11.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:10 smithi078 ceph-mon[63787]: pgmap v370: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:12.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:12 smithi136 ceph-mon[49560]: pgmap v371: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:13.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:12 smithi078 ceph-mon[63787]: pgmap v371: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:14.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:14 smithi136 ceph-mon[49560]: pgmap v372: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:51:15.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:14 smithi078 ceph-mon[63787]: pgmap v372: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:51:16.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:16 smithi136 ceph-mon[49560]: pgmap v373: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:17.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:16 smithi078 ceph-mon[63787]: pgmap v373: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:18.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:18 smithi136 ceph-mon[49560]: pgmap v374: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:51:19.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:18 smithi078 ceph-mon[63787]: pgmap v374: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:51:20.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:20 smithi136 ceph-mon[49560]: pgmap v375: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:21.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:20 smithi078 ceph-mon[63787]: pgmap v375: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:22.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:22 smithi136 ceph-mon[49560]: pgmap v376: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:23.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:22 smithi078 ceph-mon[63787]: pgmap v376: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:23.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:23 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:51:24.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:23 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:51:24.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:24 smithi136 ceph-mon[49560]: pgmap v377: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:51:25.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:24 smithi078 ceph-mon[63787]: pgmap v377: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:51:26.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:26 smithi136 ceph-mon[49560]: pgmap v378: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:27.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:26 smithi078 ceph-mon[63787]: pgmap v378: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:28.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:28 smithi136 ceph-mon[49560]: pgmap v379: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:51:29.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:28 smithi078 ceph-mon[63787]: pgmap v379: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:51:30.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:30 smithi136 ceph-mon[49560]: pgmap v380: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:31.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:30 smithi078 ceph-mon[63787]: pgmap v380: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:32.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:32 smithi136 ceph-mon[49560]: pgmap v381: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:33.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:32 smithi078 ceph-mon[63787]: pgmap v381: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:34.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:34 smithi136 ceph-mon[49560]: pgmap v382: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:51:35.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:34 smithi078 ceph-mon[63787]: pgmap v382: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:51:36.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:36 smithi136 ceph-mon[49560]: pgmap v383: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:37.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:36 smithi078 ceph-mon[63787]: pgmap v383: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:38.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:37 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:51:38.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:37 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:51:39.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:38 smithi078 ceph-mon[63787]: pgmap v384: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:51:39.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:38 smithi136 ceph-mon[49560]: pgmap v384: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:51:40.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:40 smithi136 ceph-mon[49560]: pgmap v385: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:41.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:40 smithi078 ceph-mon[63787]: pgmap v385: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:42.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:42 smithi136 ceph-mon[49560]: pgmap v386: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:43.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:42 smithi078 ceph-mon[63787]: pgmap v386: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:44.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:44 smithi136 ceph-mon[49560]: pgmap v387: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:51:45.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:44 smithi078 ceph-mon[63787]: pgmap v387: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:51:46.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:46 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:51:46.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:46 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T09:51:47.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:47 smithi136 ceph-mon[49560]: pgmap v388: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:47.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:47 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:51:47.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:47 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:51:47.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:47 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:51:47.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:47 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:51:47.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:47 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:51:47.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:47 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:51:47.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:47 smithi078 ceph-mon[63787]: pgmap v388: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:47.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:47 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:51:47.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:47 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:51:47.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:47 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T09:51:47.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:47 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T09:51:47.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:47 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:51:47.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:47 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' 2024-09-17T09:51:48.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:48 smithi136 ceph-mon[49560]: pgmap v389: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:51:49.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:48 smithi078 ceph-mon[63787]: pgmap v389: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:51:50.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:50 smithi136 ceph-mon[49560]: pgmap v390: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:51.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:50 smithi078 ceph-mon[63787]: pgmap v390: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:52.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:52 smithi136 ceph-mon[49560]: pgmap v391: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:53.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:52 smithi078 ceph-mon[63787]: pgmap v391: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:53.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:53 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:51:54.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:53 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:51:54.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:54 smithi136 ceph-mon[49560]: pgmap v392: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:51:55.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:54 smithi078 ceph-mon[63787]: pgmap v392: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:51:56.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:56 smithi136 ceph-mon[49560]: pgmap v393: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:57.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:56 smithi078 ceph-mon[63787]: pgmap v393: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:51:58.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:51:58 smithi136 ceph-mon[49560]: pgmap v394: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:51:59.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:51:58 smithi078 ceph-mon[63787]: pgmap v394: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:52:00.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:00 smithi136 ceph-mon[49560]: pgmap v395: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:52:01.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:00 smithi078 ceph-mon[63787]: pgmap v395: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:52:02.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:02 smithi136 ceph-mon[49560]: pgmap v396: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:52:03.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:02 smithi078 ceph-mon[63787]: pgmap v396: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:52:04.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:04 smithi136 ceph-mon[49560]: pgmap v397: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:52:05.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:04 smithi078 ceph-mon[63787]: pgmap v397: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:52:06.510 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -e sha1=e646f72961223a04d41eb40dad875d1628d78516 -- bash -c 'ceph orch ps' 2024-09-17T09:52:06.817 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:06 smithi078 ceph-mon[63787]: pgmap v398: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:52:06.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:06 smithi136 ceph-mon[49560]: pgmap v398: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:52:08.071 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:07 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:52:08.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:07 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:52:08.524 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T09:52:08.525 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (12m) 90s ago 21m 18.2M - 0.27.0 11f11916f8cd 97db2373ce92 2024-09-17T09:52:08.525 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (9m) 90s ago 21m 6899k - 19.3.0-4963-ge646f729 f271290a0950 da528c5ca445 2024-09-17T09:52:08.525 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi136 smithi136 running (9m) 116s ago 21m 6899k - 19.3.0-4963-ge646f729 f271290a0950 909f405348cc 2024-09-17T09:52:08.525 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (96s) 90s ago 21m 44.6M - 10.4.0 c8b91775d855 bb0a24f1977c 2024-09-17T09:52:08.525 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.jdrsup smithi078 running (2m) 90s ago 15m 34.4M - 19.3.0-4963-ge646f729 f271290a0950 809abd2f990d 2024-09-17T09:52:08.525 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi136.smslff smithi136 running (2m) 116s ago 15m 15.5M - 19.3.0-4963-ge646f729 f271290a0950 516254567aa2 2024-09-17T09:52:08.525 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rsqofi smithi078 *:8443,9283,8765 running (11m) 90s ago 22m 533M - 19.3.0-4963-ge646f729 f271290a0950 7dda3108c9a0 2024-09-17T09:52:08.525 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi136.hicmlm smithi136 *:8443,9283,8765 running (10m) 116s ago 20m 457M - 19.3.0-4963-ge646f729 f271290a0950 fc518b58e07a 2024-09-17T09:52:08.525 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (10m) 90s ago 23m 67.9M 2048M 19.3.0-4963-ge646f729 f271290a0950 3ead101d3db1 2024-09-17T09:52:08.525 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi136 smithi136 running (9m) 116s ago 20m 49.6M 2048M 19.3.0-4963-ge646f729 f271290a0950 4fce88842b79 2024-09-17T09:52:08.525 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.tcwbvi smithi078 *:2049 running (2m) 90s ago 15m 67.3M - 5.9 f271290a0950 d399aeec32cb 2024-09-17T09:52:08.525 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi136.regjvc smithi136 *:2049 running (2m) 116s ago 15m 19.8M - 5.9 f271290a0950 65fc579b5d9f 2024-09-17T09:52:08.526 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (12m) 90s ago 21m 10.9M - 1.7.0 72c9c2088986 52dad3adeeaa 2024-09-17T09:52:08.526 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi136 smithi136 *:9100 running (11m) 116s ago 20m 12.5M - 1.7.0 72c9c2088986 f2bc5856e35a 2024-09-17T09:52:08.526 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (8m) 90s ago 20m 434M 1936M 19.3.0-4963-ge646f729 f271290a0950 1ea81cc809b8 2024-09-17T09:52:08.526 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (7m) 90s ago 19m 402M 1936M 19.3.0-4963-ge646f729 f271290a0950 eda32560334b 2024-09-17T09:52:08.526 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (6m) 90s ago 19m 362M 1936M 19.3.0-4963-ge646f729 f271290a0950 83a32ad7c358 2024-09-17T09:52:08.526 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (5m) 90s ago 18m 344M 1936M 19.3.0-4963-ge646f729 f271290a0950 e2cd18998d2b 2024-09-17T09:52:08.526 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi136 running (5m) 116s ago 18m 371M 2704M 19.3.0-4963-ge646f729 f271290a0950 e2a18ad94664 2024-09-17T09:52:08.526 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi136 running (4m) 116s ago 17m 335M 2704M 19.3.0-4963-ge646f729 f271290a0950 df9b6954ce0f 2024-09-17T09:52:08.526 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi136 running (3m) 116s ago 17m 295M 2704M 19.3.0-4963-ge646f729 f271290a0950 c477e7043301 2024-09-17T09:52:08.526 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi136 running (3m) 116s ago 17m 290M 2704M 19.3.0-4963-ge646f729 f271290a0950 6c9da904d39c 2024-09-17T09:52:08.526 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (10m) 90s ago 20m 58.2M - 2.51.0 1d3b7f56885b 6e5e8c721eb0 2024-09-17T09:52:09.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:08 smithi078 ceph-mon[63787]: from='client.34590 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:52:09.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:08 smithi078 ceph-mon[63787]: pgmap v399: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:52:09.153 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -e sha1=e646f72961223a04d41eb40dad875d1628d78516 -- bash -c 'ceph orch upgrade status' 2024-09-17T09:52:09.225 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:08 smithi136 ceph-mon[49560]: from='client.34590 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:52:09.226 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:08 smithi136 ceph-mon[49560]: pgmap v399: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:52:10.782 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:10 smithi078 ceph-mon[63787]: pgmap v400: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:52:10.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:10 smithi136 ceph-mon[49560]: pgmap v400: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:52:11.255 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:52:11.255 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": null, 2024-09-17T09:52:11.255 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": false, 2024-09-17T09:52:11.255 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "", 2024-09-17T09:52:11.255 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-09-17T09:52:11.255 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": null, 2024-09-17T09:52:11.255 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "", 2024-09-17T09:52:11.255 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-17T09:52:11.255 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:52:11.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:11 smithi078 ceph-mon[63787]: from='client.34594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:52:11.917 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -e sha1=e646f72961223a04d41eb40dad875d1628d78516 -- bash -c 'ceph health detail' 2024-09-17T09:52:11.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:11 smithi136 ceph-mon[49560]: from='client.34594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:52:12.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:12 smithi078 ceph-mon[63787]: pgmap v401: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:52:12.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:12 smithi136 ceph-mon[49560]: pgmap v401: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:52:14.019 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-17T09:52:14.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:14 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/1990932759' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:52:14.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:14 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/1990932759' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T09:52:14.732 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -e sha1=e646f72961223a04d41eb40dad875d1628d78516 -- bash -c 'ceph versions' 2024-09-17T09:52:15.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:15 smithi078 ceph-mon[63787]: pgmap v402: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:52:15.475 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:15 smithi136 ceph-mon[49560]: pgmap v402: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:52:16.805 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-17T09:52:16.806 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-17T09:52:16.806 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:52:16.806 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:52:16.806 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-17T09:52:16.806 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:52:16.807 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:52:16.807 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-17T09:52:16.807 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 8 2024-09-17T09:52:16.807 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:52:16.807 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-17T09:52:16.807 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 2 2024-09-17T09:52:16.807 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:52:16.807 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-17T09:52:16.807 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)": 14 2024-09-17T09:52:16.807 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-17T09:52:16.807 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-17T09:52:16.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:16 smithi136 ceph-mon[49560]: pgmap v403: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:52:17.097 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:16 smithi078 ceph-mon[63787]: pgmap v403: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:52:17.494 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -e sha1=e646f72961223a04d41eb40dad875d1628d78516 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-09-17T09:52:17.790 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:17 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/1339005629' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:52:17.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:17 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/1339005629' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:52:18.770 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:18 smithi078 ceph-mon[63787]: pgmap v404: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:52:18.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:18 smithi136 ceph-mon[49560]: pgmap v404: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:52:19.598 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-17T09:52:19.910 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:19 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/2015559308' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:52:19.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:19 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/2015559308' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:52:20.269 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -e sha1=e646f72961223a04d41eb40dad875d1628d78516 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-09-17T09:52:20.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:20 smithi078 ceph-mon[63787]: pgmap v405: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:52:20.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:20 smithi136 ceph-mon[49560]: pgmap v405: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:52:22.381 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4963-ge646f729 (e646f72961223a04d41eb40dad875d1628d78516) squid (dev)" 2024-09-17T09:52:22.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:22 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/2246512418' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:52:22.749 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:22 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/2246512418' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T09:52:23.068 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-09-17T09:52:23.073 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-09-17T09:52:23.073 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-17T09:52:23.669 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:23 smithi078 ceph-mon[63787]: pgmap v406: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:52:23.669 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:23 smithi078 ceph-mon[63787]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:52:23.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:23 smithi136 ceph-mon[49560]: pgmap v406: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:52:23.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:23 smithi136 ceph-mon[49560]: from='mgr.34104 172.21.15.78:0/1365682724' entity='mgr.smithi078.rsqofi' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T09:52:24.707 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:24 smithi078 ceph-mon[63787]: pgmap v407: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:52:24.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:24 smithi136 ceph-mon[49560]: pgmap v407: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:52:25.198 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-17T09:52:25.198 INFO:teuthology.orchestra.run.smithi078.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-17T09:29:53.798273Z", "last_refresh": "2024-09-17T09:50:37.718484Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-17T09:29:47.463665Z", "last_refresh": "2024-09-17T09:50:11.671487Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-09-17T09:29:51.372259Z", "last_refresh": "2024-09-17T09:50:37.718622Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-17T09:36:28.753750Z", "last_refresh": "2024-09-17T09:50:11.672092Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-17T09:29:45.312225Z", "last_refresh": "2024-09-17T09:50:11.671597Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi078:172.21.15.78=smithi078", "smithi136:172.21.15.136=smithi136"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-17T09:30:35.435397Z", "last_refresh": "2024-09-17T09:50:11.671666Z", "running": 2, "size": 2}}, {"events": ["2024-09-17T09:51:47.006561Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-09-17T09:36:48.928165Z", "last_refresh": "2024-09-17T09:50:11.672162Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-17T09:29:52.565464Z", "last_refresh": "2024-09-17T09:50:11.671732Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407", "last_refresh": "2024-09-17T09:50:11.671796Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-17T09:29:49.143959Z", "last_refresh": "2024-09-17T09:50:37.718757Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-17T09:52:25.822 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:25 smithi078 ceph-mon[63787]: from='client.34614 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T09:52:25.850 INFO:tasks.cephadm:nfs.foo has 2/2 2024-09-17T09:52:25.850 DEBUG:teuthology.parallel:result is None 2024-09-17T09:52:25.851 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-17T09:52:25.860 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-17T09:52:25.860 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-09-17T09:52:25.888 INFO:teuthology.orchestra.run.smithi078.stderr:+ umount /mnt/foo 2024-09-17T09:52:25.918 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-17T09:52:25.928 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-17T09:52:25.928 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-09-17T09:52:25.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:25 smithi136 ceph-mon[49560]: from='client.34614 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T09:52:26.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:26 smithi078 ceph-mon[63787]: pgmap v408: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:52:26.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:26 smithi136 ceph-mon[49560]: pgmap v408: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:52:27.874 INFO:teuthology.orchestra.run.smithi078.stdout: "foo" 2024-09-17T09:52:28.519 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs export ls foo --detailed' 2024-09-17T09:52:28.818 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:28 smithi078 ceph-mon[63787]: from='client.34618 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:52:28.818 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:28 smithi078 ceph-mon[63787]: pgmap v409: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:52:28.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:28 smithi136 ceph-mon[49560]: from='client.34618 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:52:28.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:28 smithi136 ceph-mon[49560]: pgmap v409: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:52:30.757 INFO:teuthology.orchestra.run.smithi078.stdout:[ 2024-09-17T09:52:30.757 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-09-17T09:52:30.757 INFO:teuthology.orchestra.run.smithi078.stdout: "access_type": "RW", 2024-09-17T09:52:30.757 INFO:teuthology.orchestra.run.smithi078.stdout: "clients": [], 2024-09-17T09:52:30.757 INFO:teuthology.orchestra.run.smithi078.stdout: "cluster_id": "foo", 2024-09-17T09:52:30.757 INFO:teuthology.orchestra.run.smithi078.stdout: "export_id": 1, 2024-09-17T09:52:30.757 INFO:teuthology.orchestra.run.smithi078.stdout: "fsal": { 2024-09-17T09:52:30.757 INFO:teuthology.orchestra.run.smithi078.stdout: "fs_name": "foofs", 2024-09-17T09:52:30.758 INFO:teuthology.orchestra.run.smithi078.stdout: "name": "CEPH", 2024-09-17T09:52:30.758 INFO:teuthology.orchestra.run.smithi078.stdout: "user_id": "nfs.foo.1" 2024-09-17T09:52:30.758 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-17T09:52:30.758 INFO:teuthology.orchestra.run.smithi078.stdout: "path": "/", 2024-09-17T09:52:30.758 INFO:teuthology.orchestra.run.smithi078.stdout: "protocols": [ 2024-09-17T09:52:30.758 INFO:teuthology.orchestra.run.smithi078.stdout: 4 2024-09-17T09:52:30.758 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-17T09:52:30.758 INFO:teuthology.orchestra.run.smithi078.stdout: "pseudo": "/fake", 2024-09-17T09:52:30.758 INFO:teuthology.orchestra.run.smithi078.stdout: "security_label": true, 2024-09-17T09:52:30.758 INFO:teuthology.orchestra.run.smithi078.stdout: "squash": "none", 2024-09-17T09:52:30.758 INFO:teuthology.orchestra.run.smithi078.stdout: "transports": [ 2024-09-17T09:52:30.758 INFO:teuthology.orchestra.run.smithi078.stdout: "TCP" 2024-09-17T09:52:30.758 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-09-17T09:52:30.758 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-17T09:52:30.759 INFO:teuthology.orchestra.run.smithi078.stdout:] 2024-09-17T09:52:30.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:30 smithi136 ceph-mon[49560]: pgmap v410: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:52:31.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:30 smithi078 ceph-mon[63787]: pgmap v410: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T09:52:31.433 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- bash -c 'rados -p .nfs --all ls -' 2024-09-17T09:52:31.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:31 smithi136 ceph-mon[49560]: from='client.34622 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:52:31.978 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:31 smithi078 ceph-mon[63787]: from='client.34622 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T09:52:32.757 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:32 smithi078 ceph-mon[63787]: pgmap v411: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:52:32.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:32 smithi136 ceph-mon[49560]: pgmap v411: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:52:33.327 INFO:teuthology.orchestra.run.smithi078.stdout:foo rec-0000000000000004:nfs.foo.1 2024-09-17T09:52:33.327 INFO:teuthology.orchestra.run.smithi078.stdout:foo grace 2024-09-17T09:52:33.327 INFO:teuthology.orchestra.run.smithi078.stdout:foo rec-0000000000000004:nfs.foo.0 2024-09-17T09:52:33.327 INFO:teuthology.orchestra.run.smithi078.stdout:foo conf-nfs.foo 2024-09-17T09:52:33.327 INFO:teuthology.orchestra.run.smithi078.stdout:foo export-1 2024-09-17T09:52:33.976 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:33 smithi136 ceph-mon[49560]: mgrmap e44: smithi078.rsqofi(active, since 10m), standbys: smithi136.hicmlm 2024-09-17T09:52:34.023 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:33 smithi078 ceph-mon[63787]: mgrmap e44: smithi078.rsqofi(active, since 10m), standbys: smithi136.hicmlm 2024-09-17T09:52:34.052 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-17T09:52:34.062 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-17T09:52:34.062 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 -- bash -c 'set -ex 2024-09-17T09:52:34.063 DEBUG:teuthology.orchestra.run.smithi078:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-09-17T09:52:34.063 DEBUG:teuthology.orchestra.run.smithi078:> ' 2024-09-17T09:52:34.975 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:34 smithi136 ceph-mon[49560]: pgmap v412: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T09:52:35.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:34 smithi078 ceph-mon[63787]: pgmap v412: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T09:52:35.833 INFO:teuthology.orchestra.run.smithi078.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-09-17T09:52:36.058 INFO:teuthology.orchestra.run.smithi078.stderr:+ [[ 8 -gt 2 ]] 2024-09-17T09:52:36.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:36 smithi078 ceph-mon[63787]: from='client.? 172.21.15.78:0/2180854483' entity='client.admin' cmd={"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"} : dispatch 2024-09-17T09:52:36.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:36 smithi136 ceph-mon[49560]: from='client.? 172.21.15.78:0/2180854483' entity='client.admin' cmd={"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"} : dispatch 2024-09-17T09:52:36.769 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-17T09:52:36.779 INFO:tasks.cephadm:Teardown begin 2024-09-17T09:52:36.779 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-17T09:52:36.806 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-17T09:52:36.839 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-17T09:52:36.839 DEBUG:teuthology.orchestra.run.smithi078:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-17T09:52:36.862 DEBUG:teuthology.orchestra.run.smithi136:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-17T09:52:36.897 INFO:tasks.cephadm:Stopping all daemons... 2024-09-17T09:52:36.898 INFO:tasks.cephadm.mon.smithi078:Stopping mon.smithi078... 2024-09-17T09:52:36.898 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@mon.smithi078 2024-09-17T09:52:37.096 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:36 smithi078 systemd[1]: Stopping Ceph mon.smithi078 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:52:37.476 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:37 smithi136 ceph-mon[49560]: pgmap v413: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:52:37.559 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:37 smithi078 ceph-mon[63787]: pgmap v413: 97 pgs: 97 active+clean; 233 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T09:52:37.559 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:37 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi078[63783]: 2024-09-17T09:52:37.197+0000 7f49b4817640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi078 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:52:37.559 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:37 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi078[63783]: 2024-09-17T09:52:37.197+0000 7f49b4817640 -1 mon.smithi078@0(leader) e3 *** Got Signal Terminated *** 2024-09-17T09:52:37.559 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:37 smithi078 podman[93625]: 2024-09-17 09:52:37.259149412 +0000 UTC m=+0.170004045 container died 3ead101d3db1d111420cb146309cd04437cd607a750073cf14fdb72f34049737 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi078, org.label-schema.license=GPLv2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-17T09:52:37.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:37 smithi078 podman[93625]: 2024-09-17 09:52:37.656494576 +0000 UTC m=+0.567349208 container cleanup 3ead101d3db1d111420cb146309cd04437cd607a750073cf14fdb72f34049737 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi078, GIT_BRANCH=HEAD, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-17T09:52:37.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 17 09:52:37 smithi078 bash[93625]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi078 2024-09-17T09:52:38.094 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@mon.smithi078.service' 2024-09-17T09:52:38.146 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T09:52:38.146 INFO:tasks.cephadm.mon.smithi078:Stopped mon.smithi078 2024-09-17T09:52:38.146 INFO:tasks.cephadm.mon.smithi136:Stopping mon.smithi136... 2024-09-17T09:52:38.147 DEBUG:teuthology.orchestra.run.smithi136:> sudo systemctl stop ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@mon.smithi136 2024-09-17T09:52:38.467 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:38 smithi136 systemd[1]: Stopping Ceph mon.smithi136 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:52:38.725 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:38 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi136[49556]: 2024-09-17T09:52:38.464+0000 7f78fefcf640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi136 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:52:38.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:38 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi136[49556]: 2024-09-17T09:52:38.464+0000 7f78fefcf640 -1 mon.smithi136@1(peon) e3 *** Got Signal Terminated *** 2024-09-17T09:52:38.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:38 smithi136 podman[68087]: 2024-09-17 09:52:38.680073051 +0000 UTC m=+0.341150567 container stop 4fce88842b79c82363aecf01ca6b4c6264d292d49e481ad99acacb35a9646d5a (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi136, GIT_BRANCH=HEAD, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731) 2024-09-17T09:52:38.726 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:38 smithi136 podman[68087]: 2024-09-17 09:52:38.705554215 +0000 UTC m=+0.366631731 container died 4fce88842b79c82363aecf01ca6b4c6264d292d49e481ad99acacb35a9646d5a (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi136, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True) 2024-09-17T09:52:39.401 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:39 smithi136 podman[68087]: 2024-09-17 09:52:39.19070277 +0000 UTC m=+0.851780328 container cleanup 4fce88842b79c82363aecf01ca6b4c6264d292d49e481ad99acacb35a9646d5a (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi136, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72) 2024-09-17T09:52:39.401 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:39 smithi136 bash[68087]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi136 2024-09-17T09:52:39.656 INFO:journalctl@ceph.mon.smithi136.smithi136.stdout:Sep 17 09:52:39 smithi136 podman[68099]: 2024-09-17 09:52:39.400509831 +0000 UTC m=+0.710933072 container remove 4fce88842b79c82363aecf01ca6b4c6264d292d49e481ad99acacb35a9646d5a (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-mon-smithi136, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731) 2024-09-17T09:52:39.661 DEBUG:teuthology.orchestra.run.smithi136:> sudo pkill -f 'journalctl -f -n 0 -u ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@mon.smithi136.service' 2024-09-17T09:52:39.709 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T09:52:39.709 INFO:tasks.cephadm.mon.smithi136:Stopped mon.smithi136 2024-09-17T09:52:39.710 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-17T09:52:39.710 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.0 2024-09-17T09:52:40.037 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:52:39 smithi078 systemd[1]: Stopping Ceph osd.0 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:52:40.346 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:52:40 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0[68716]: 2024-09-17T09:52:40.035+0000 7f934defd640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:52:40.346 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:52:40 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0[68716]: 2024-09-17T09:52:40.035+0000 7f934defd640 -1 osd.0 161 *** Got signal Terminated *** 2024-09-17T09:52:40.346 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:52:40 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0[68716]: 2024-09-17T09:52:40.035+0000 7f934defd640 -1 osd.0 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:52:45.955 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:52:45 smithi078 podman[93759]: 2024-09-17 09:52:45.637498305 +0000 UTC m=+5.739257400 container died 1ea81cc809b834b415f1f06ca195a8ec0906e0a1e7c5565938ad2d3e4765a8a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-17T09:52:46.305 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:52:46 smithi078 podman[93759]: 2024-09-17 09:52:46.052247655 +0000 UTC m=+6.154006742 container cleanup 1ea81cc809b834b415f1f06ca195a8ec0906e0a1e7c5565938ad2d3e4765a8a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-17T09:52:46.305 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:52:46 smithi078 bash[93759]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0 2024-09-17T09:52:46.596 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:52:46 smithi078 podman[93790]: 2024-09-17 09:52:46.304022576 +0000 UTC m=+0.662635396 container remove 1ea81cc809b834b415f1f06ca195a8ec0906e0a1e7c5565938ad2d3e4765a8a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-17T09:52:47.017 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:52:46 smithi078 podman[93878]: 2024-09-17 09:52:46.629203389 +0000 UTC m=+0.016262250 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:52:47.017 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:52:46 smithi078 podman[93878]: 2024-09-17 09:52:46.76312471 +0000 UTC m=+0.150183572 container create 4f87a54acebd0cada5306a24e8323de704829849f8bb59c31a4398adecff426f (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-17T09:52:47.017 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:52:47 smithi078 podman[93878]: 2024-09-17 09:52:47.015642792 +0000 UTC m=+0.402701648 container init 4f87a54acebd0cada5306a24e8323de704829849f8bb59c31a4398adecff426f (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-17T09:52:47.346 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:52:47 smithi078 podman[93878]: 2024-09-17 09:52:47.019373012 +0000 UTC m=+0.406431873 container start 4f87a54acebd0cada5306a24e8323de704829849f8bb59c31a4398adecff426f (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:52:47.347 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:52:47 smithi078 podman[93878]: 2024-09-17 09:52:47.140731951 +0000 UTC m=+0.527790805 container attach 4f87a54acebd0cada5306a24e8323de704829849f8bb59c31a4398adecff426f (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-17T09:52:47.347 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 17 09:52:47 smithi078 podman[93878]: 2024-09-17 09:52:47.221802748 +0000 UTC m=+0.608861606 container died 4f87a54acebd0cada5306a24e8323de704829849f8bb59c31a4398adecff426f (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72) 2024-09-17T09:52:47.920 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.0.service' 2024-09-17T09:52:47.960 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T09:52:47.960 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-17T09:52:47.960 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-17T09:52:47.960 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.1 2024-09-17T09:52:48.307 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:52:48 smithi078 systemd[1]: Stopping Ceph osd.1 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:52:48.596 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:52:48 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1[71058]: 2024-09-17T09:52:48.305+0000 7fb9e7661640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:52:48.596 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:52:48 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1[71058]: 2024-09-17T09:52:48.305+0000 7fb9e7661640 -1 osd.1 161 *** Got signal Terminated *** 2024-09-17T09:52:48.596 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:52:48 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1[71058]: 2024-09-17T09:52:48.305+0000 7fb9e7661640 -1 osd.1 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:52:54.174 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:52:53 smithi078 podman[93980]: 2024-09-17 09:52:53.885003584 +0000 UTC m=+5.696185173 container died eda32560334bf3fa80e0714a87e3774c929c17f1e9f04ab098c87a138d64648e (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True) 2024-09-17T09:52:54.566 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:52:54 smithi078 podman[93980]: 2024-09-17 09:52:54.247305761 +0000 UTC m=+6.058487345 container cleanup eda32560334bf3fa80e0714a87e3774c929c17f1e9f04ab098c87a138d64648e (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-17T09:52:54.566 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:52:54 smithi078 bash[93980]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1 2024-09-17T09:52:54.846 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:52:54 smithi078 podman[93997]: 2024-09-17 09:52:54.565221395 +0000 UTC m=+0.673863913 container remove eda32560334bf3fa80e0714a87e3774c929c17f1e9f04ab098c87a138d64648e (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-17T09:52:55.278 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:52:54 smithi078 podman[94085]: 2024-09-17 09:52:54.818106168 +0000 UTC m=+0.016602559 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:52:55.278 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:52:54 smithi078 podman[94085]: 2024-09-17 09:52:54.949704123 +0000 UTC m=+0.148200504 container create 2a6116e7e481ba83877bcc7640780a8b5a2a920528bf8513437ef807c8f0c255 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True) 2024-09-17T09:52:55.278 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:52:55 smithi078 podman[94085]: 2024-09-17 09:52:55.160322361 +0000 UTC m=+0.358818743 container init 2a6116e7e481ba83877bcc7640780a8b5a2a920528bf8513437ef807c8f0c255 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-17T09:52:55.278 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:52:55 smithi078 podman[94085]: 2024-09-17 09:52:55.163831282 +0000 UTC m=+0.362327664 container start 2a6116e7e481ba83877bcc7640780a8b5a2a920528bf8513437ef807c8f0c255 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T09:52:55.278 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:52:55 smithi078 podman[94085]: 2024-09-17 09:52:55.277226989 +0000 UTC m=+0.475723388 container attach 2a6116e7e481ba83877bcc7640780a8b5a2a920528bf8513437ef807c8f0c255 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.vendor=CentOS) 2024-09-17T09:52:55.596 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 17 09:52:55 smithi078 podman[94085]: 2024-09-17 09:52:55.351736353 +0000 UTC m=+0.550232740 container died 2a6116e7e481ba83877bcc7640780a8b5a2a920528bf8513437ef807c8f0c255 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-17T09:52:56.077 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.1.service' 2024-09-17T09:52:56.122 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T09:52:56.122 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-17T09:52:56.122 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-17T09:52:56.123 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.2 2024-09-17T09:52:56.346 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:52:56 smithi078 systemd[1]: Stopping Ceph osd.2 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:52:56.846 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:52:56 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2[73492]: 2024-09-17T09:52:56.442+0000 7f124f9df640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:52:56.846 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:52:56 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2[73492]: 2024-09-17T09:52:56.442+0000 7f124f9df640 -1 osd.2 161 *** Got signal Terminated *** 2024-09-17T09:52:56.847 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:52:56 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2[73492]: 2024-09-17T09:52:56.442+0000 7f124f9df640 -1 osd.2 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:53:02.153 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:53:01 smithi078 podman[94187]: 2024-09-17 09:53:01.865299212 +0000 UTC m=+5.518821896 container died 83a32ad7c358ab77288d7ebbb4c5b7abc55749669b2d48e6265ff6721c955cd1 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.schema-version=1.0) 2024-09-17T09:53:02.528 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:53:02 smithi078 podman[94187]: 2024-09-17 09:53:02.233940127 +0000 UTC m=+5.887462808 container cleanup 83a32ad7c358ab77288d7ebbb4c5b7abc55749669b2d48e6265ff6721c955cd1 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-17T09:53:02.528 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:53:02 smithi078 bash[94187]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2 2024-09-17T09:53:02.839 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:53:02 smithi078 podman[94207]: 2024-09-17 09:53:02.527181577 +0000 UTC m=+0.663634354 container remove 83a32ad7c358ab77288d7ebbb4c5b7abc55749669b2d48e6265ff6721c955cd1 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-17T09:53:03.093 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:53:02 smithi078 podman[94292]: 2024-09-17 09:53:02.741648549 +0000 UTC m=+0.016538705 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:53:03.093 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:53:02 smithi078 podman[94292]: 2024-09-17 09:53:02.869840066 +0000 UTC m=+0.144730255 container create cd368dacfcf00d3740cd0c469a79c49679d683cb614f34289298823d26d9950b (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2-deactivate, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-17T09:53:03.093 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:53:03 smithi078 podman[94292]: 2024-09-17 09:53:03.088752323 +0000 UTC m=+0.363642470 container init cd368dacfcf00d3740cd0c469a79c49679d683cb614f34289298823d26d9950b (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72) 2024-09-17T09:53:03.347 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:53:03 smithi078 podman[94292]: 2024-09-17 09:53:03.092170968 +0000 UTC m=+0.367061116 container start cd368dacfcf00d3740cd0c469a79c49679d683cb614f34289298823d26d9950b (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T09:53:03.347 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:53:03 smithi078 podman[94292]: 2024-09-17 09:53:03.205555274 +0000 UTC m=+0.480445419 container attach cd368dacfcf00d3740cd0c469a79c49679d683cb614f34289298823d26d9950b (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2-deactivate, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-17T09:53:03.347 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 17 09:53:03 smithi078 podman[94292]: 2024-09-17 09:53:03.287253559 +0000 UTC m=+0.562143714 container died cd368dacfcf00d3740cd0c469a79c49679d683cb614f34289298823d26d9950b (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-2-deactivate, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-17T09:53:03.948 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.2.service' 2024-09-17T09:53:03.987 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T09:53:03.987 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-17T09:53:03.987 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-17T09:53:03.987 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.3 2024-09-17T09:53:04.322 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:53:04 smithi078 systemd[1]: Stopping Ceph osd.3 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:53:04.596 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:53:04 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3[75868]: 2024-09-17T09:53:04.320+0000 7f7a928f3640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:53:04.596 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:53:04 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3[75868]: 2024-09-17T09:53:04.320+0000 7f7a928f3640 -1 osd.3 161 *** Got signal Terminated *** 2024-09-17T09:53:04.597 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:53:04 smithi078 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3[75868]: 2024-09-17T09:53:04.320+0000 7f7a928f3640 -1 osd.3 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:53:07.226 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:06 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:06.893+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:40.904179+0000 front 2024-09-17T09:52:40.904227+0000 (oldest deadline 2024-09-17T09:53:06.204000+0000) 2024-09-17T09:53:07.976 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:53:07 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4[53412]: 2024-09-17T09:53:07.679+0000 7f8322111640 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.450193+0000 front 2024-09-17T09:52:44.450180+0000 (oldest deadline 2024-09-17T09:53:07.349857+0000) 2024-09-17T09:53:07.976 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:07 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:07.876+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:40.904179+0000 front 2024-09-17T09:52:40.904227+0000 (oldest deadline 2024-09-17T09:53:06.204000+0000) 2024-09-17T09:53:08.683 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:08 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:08.349+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:08.976 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:53:08 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4[53412]: 2024-09-17T09:53:08.682+0000 7f8322111640 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.450193+0000 front 2024-09-17T09:52:44.450180+0000 (oldest deadline 2024-09-17T09:53:07.349857+0000) 2024-09-17T09:53:08.976 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:08 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:08.876+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:40.904179+0000 front 2024-09-17T09:52:40.904227+0000 (oldest deadline 2024-09-17T09:53:06.204000+0000) 2024-09-17T09:53:09.475 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:09 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:09.199+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.814583+0000 front 2024-09-17T09:52:44.814673+0000 (oldest deadline 2024-09-17T09:53:08.314503+0000) 2024-09-17T09:53:09.476 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:09 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:09.339+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:09.975 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:53:09 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4[53412]: 2024-09-17T09:53:09.720+0000 7f8322111640 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.450193+0000 front 2024-09-17T09:52:44.450180+0000 (oldest deadline 2024-09-17T09:53:07.349857+0000) 2024-09-17T09:53:09.976 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:09 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:09.834+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:40.904179+0000 front 2024-09-17T09:52:40.904227+0000 (oldest deadline 2024-09-17T09:53:06.204000+0000) 2024-09-17T09:53:10.239 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:53:09 smithi078 podman[94411]: 2024-09-17 09:53:09.931220488 +0000 UTC m=+5.723094961 container died e2cd18998d2b4d67879058627c83bbd4cd393d9f4692e7de722d49815708f82e (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T09:53:10.475 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:10 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:10.160+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.814583+0000 front 2024-09-17T09:52:44.814673+0000 (oldest deadline 2024-09-17T09:53:08.314503+0000) 2024-09-17T09:53:10.476 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:10 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:10.302+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:10.530 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:53:10 smithi078 podman[94411]: 2024-09-17 09:53:10.311934554 +0000 UTC m=+6.103809027 container cleanup e2cd18998d2b4d67879058627c83bbd4cd393d9f4692e7de722d49815708f82e (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:53:10.531 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:53:10 smithi078 bash[94411]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3 2024-09-17T09:53:10.846 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:53:10 smithi078 podman[94431]: 2024-09-17 09:53:10.529752805 +0000 UTC m=+0.599553076 container remove e2cd18998d2b4d67879058627c83bbd4cd393d9f4692e7de722d49815708f82e (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-17T09:53:10.975 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:53:10 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4[53412]: 2024-09-17T09:53:10.696+0000 7f8322111640 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.450193+0000 front 2024-09-17T09:52:44.450180+0000 (oldest deadline 2024-09-17T09:53:07.349857+0000) 2024-09-17T09:53:10.976 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:10 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:10.830+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:40.904179+0000 front 2024-09-17T09:52:40.904227+0000 (oldest deadline 2024-09-17T09:53:06.204000+0000) 2024-09-17T09:53:11.117 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:53:10 smithi078 podman[94514]: 2024-09-17 09:53:10.752706863 +0000 UTC m=+0.016049337 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:53:11.118 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:53:10 smithi078 podman[94514]: 2024-09-17 09:53:10.872249401 +0000 UTC m=+0.135591862 container create 2479884174b3a994200a0a6d116d6e85635e100dae308f08260425a66eaecdda (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-17T09:53:11.118 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:53:11 smithi078 podman[94514]: 2024-09-17 09:53:11.116592527 +0000 UTC m=+0.379935004 container init 2479884174b3a994200a0a6d116d6e85635e100dae308f08260425a66eaecdda (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-17T09:53:11.475 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:11 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:11.152+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.814583+0000 front 2024-09-17T09:52:44.814673+0000 (oldest deadline 2024-09-17T09:53:08.314503+0000) 2024-09-17T09:53:11.476 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:11 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:11.254+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:11.597 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:53:11 smithi078 podman[94514]: 2024-09-17 09:53:11.119870524 +0000 UTC m=+0.383212986 container start 2479884174b3a994200a0a6d116d6e85635e100dae308f08260425a66eaecdda (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-17T09:53:11.597 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:53:11 smithi078 podman[94514]: 2024-09-17 09:53:11.199902575 +0000 UTC m=+0.463245059 container attach 2479884174b3a994200a0a6d116d6e85635e100dae308f08260425a66eaecdda (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-17T09:53:11.597 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 17 09:53:11 smithi078 podman[94514]: 2024-09-17 09:53:11.308446657 +0000 UTC m=+0.571789118 container died 2479884174b3a994200a0a6d116d6e85635e100dae308f08260425a66eaecdda (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-3-deactivate, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-17T09:53:11.974 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.3.service' 2024-09-17T09:53:11.975 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:53:11 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4[53412]: 2024-09-17T09:53:11.654+0000 7f8322111640 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.450193+0000 front 2024-09-17T09:52:44.450180+0000 (oldest deadline 2024-09-17T09:53:07.349857+0000) 2024-09-17T09:53:11.976 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:11 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:11.800+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:40.904179+0000 front 2024-09-17T09:52:40.904227+0000 (oldest deadline 2024-09-17T09:53:06.204000+0000) 2024-09-17T09:53:12.014 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T09:53:12.015 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-17T09:53:12.015 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-17T09:53:12.015 DEBUG:teuthology.orchestra.run.smithi136:> sudo systemctl stop ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.4 2024-09-17T09:53:12.308 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:53:12 smithi136 systemd[1]: Stopping Ceph osd.4 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:53:12.308 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:12 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:12.263+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:12.309 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:12 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:12.187+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.814583+0000 front 2024-09-17T09:52:44.814673+0000 (oldest deadline 2024-09-17T09:53:08.314503+0000) 2024-09-17T09:53:12.675 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:53:12 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4[53412]: 2024-09-17T09:53:12.306+0000 7f8325afc640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:53:12.676 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:53:12 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4[53412]: 2024-09-17T09:53:12.306+0000 7f8325afc640 -1 osd.4 161 *** Got signal Terminated *** 2024-09-17T09:53:12.676 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:53:12 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4[53412]: 2024-09-17T09:53:12.306+0000 7f8325afc640 -1 osd.4 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:53:12.975 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:53:12 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4[53412]: 2024-09-17T09:53:12.674+0000 7f8322111640 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.450193+0000 front 2024-09-17T09:52:44.450180+0000 (oldest deadline 2024-09-17T09:53:07.349857+0000) 2024-09-17T09:53:12.976 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:12 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:12.831+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:40.904179+0000 front 2024-09-17T09:52:40.904227+0000 (oldest deadline 2024-09-17T09:53:06.204000+0000) 2024-09-17T09:53:13.476 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:13 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:13.234+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.814583+0000 front 2024-09-17T09:52:44.814673+0000 (oldest deadline 2024-09-17T09:53:08.314503+0000) 2024-09-17T09:53:13.476 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:13 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:13.222+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:13.975 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:53:13 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4[53412]: 2024-09-17T09:53:13.642+0000 7f8322111640 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.450193+0000 front 2024-09-17T09:52:44.450180+0000 (oldest deadline 2024-09-17T09:53:07.349857+0000) 2024-09-17T09:53:13.976 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:13 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:13.819+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:40.904179+0000 front 2024-09-17T09:52:40.904227+0000 (oldest deadline 2024-09-17T09:53:06.204000+0000) 2024-09-17T09:53:14.475 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:14 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:14.267+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.814583+0000 front 2024-09-17T09:52:44.814673+0000 (oldest deadline 2024-09-17T09:53:08.314503+0000) 2024-09-17T09:53:14.476 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:14 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:14.204+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:14.976 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:53:14 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4[53412]: 2024-09-17T09:53:14.669+0000 7f8322111640 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.450193+0000 front 2024-09-17T09:52:44.450180+0000 (oldest deadline 2024-09-17T09:53:07.349857+0000) 2024-09-17T09:53:14.976 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:14 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:14.811+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:40.904179+0000 front 2024-09-17T09:52:40.904227+0000 (oldest deadline 2024-09-17T09:53:06.204000+0000) 2024-09-17T09:53:14.976 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:14 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:14.811+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.205516+0000 front 2024-09-17T09:52:50.205563+0000 (oldest deadline 2024-09-17T09:53:14.305408+0000) 2024-09-17T09:53:15.476 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:15 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:15.255+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.814583+0000 front 2024-09-17T09:52:44.814673+0000 (oldest deadline 2024-09-17T09:53:08.314503+0000) 2024-09-17T09:53:15.476 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:15 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:15.200+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:15.476 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:15 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:15.200+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.810464+0000 front 2024-09-17T09:52:50.810449+0000 (oldest deadline 2024-09-17T09:53:14.910321+0000) 2024-09-17T09:53:15.976 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:53:15 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4[53412]: 2024-09-17T09:53:15.701+0000 7f8322111640 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.450193+0000 front 2024-09-17T09:52:44.450180+0000 (oldest deadline 2024-09-17T09:53:07.349857+0000) 2024-09-17T09:53:15.976 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:53:15 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4[53412]: 2024-09-17T09:53:15.701+0000 7f8322111640 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:53.250830+0000 front 2024-09-17T09:52:53.250946+0000 (oldest deadline 2024-09-17T09:53:15.550774+0000) 2024-09-17T09:53:15.976 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:15 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:15.762+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:40.904179+0000 front 2024-09-17T09:52:40.904227+0000 (oldest deadline 2024-09-17T09:53:06.204000+0000) 2024-09-17T09:53:15.976 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:15 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:15.762+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.205516+0000 front 2024-09-17T09:52:50.205563+0000 (oldest deadline 2024-09-17T09:53:14.305408+0000) 2024-09-17T09:53:16.476 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:16 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:16.268+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.814583+0000 front 2024-09-17T09:52:44.814673+0000 (oldest deadline 2024-09-17T09:53:08.314503+0000) 2024-09-17T09:53:16.476 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:16 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:16.170+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:16.476 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:16 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:16.170+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.810464+0000 front 2024-09-17T09:52:50.810449+0000 (oldest deadline 2024-09-17T09:53:14.910321+0000) 2024-09-17T09:53:17.217 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:53:16 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4[53412]: 2024-09-17T09:53:16.735+0000 7f8322111640 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.450193+0000 front 2024-09-17T09:52:44.450180+0000 (oldest deadline 2024-09-17T09:53:07.349857+0000) 2024-09-17T09:53:17.217 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:53:16 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4[53412]: 2024-09-17T09:53:16.735+0000 7f8322111640 -1 osd.4 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:53.250830+0000 front 2024-09-17T09:52:53.250946+0000 (oldest deadline 2024-09-17T09:53:15.550774+0000) 2024-09-17T09:53:17.217 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:16 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:16.806+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:40.904179+0000 front 2024-09-17T09:52:40.904227+0000 (oldest deadline 2024-09-17T09:53:06.204000+0000) 2024-09-17T09:53:17.218 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:16 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:16.806+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.205516+0000 front 2024-09-17T09:52:50.205563+0000 (oldest deadline 2024-09-17T09:53:14.305408+0000) 2024-09-17T09:53:17.476 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:17 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:17.286+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.814583+0000 front 2024-09-17T09:52:44.814673+0000 (oldest deadline 2024-09-17T09:53:08.314503+0000) 2024-09-17T09:53:17.476 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:17 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:17.286+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:51.116044+0000 front 2024-09-17T09:52:51.116104+0000 (oldest deadline 2024-09-17T09:53:17.015934+0000) 2024-09-17T09:53:17.476 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:17 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:17.214+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:17.476 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:17 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:17.214+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.810464+0000 front 2024-09-17T09:52:50.810449+0000 (oldest deadline 2024-09-17T09:53:14.910321+0000) 2024-09-17T09:53:18.198 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:53:17 smithi136 podman[68240]: 2024-09-17 09:53:17.907277923 +0000 UTC m=+5.694753638 container died e2a18ad9466438e628af3d470891ea0615e10c803666172b228adffff0ecee82 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-17T09:53:18.198 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:17 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:17.774+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:40.904179+0000 front 2024-09-17T09:52:40.904227+0000 (oldest deadline 2024-09-17T09:53:06.204000+0000) 2024-09-17T09:53:18.198 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:17 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:17.774+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.205516+0000 front 2024-09-17T09:52:50.205563+0000 (oldest deadline 2024-09-17T09:53:14.305408+0000) 2024-09-17T09:53:18.475 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:53:18 smithi136 podman[68240]: 2024-09-17 09:53:18.4153372 +0000 UTC m=+6.202812905 container cleanup e2a18ad9466438e628af3d470891ea0615e10c803666172b228adffff0ecee82 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-17T09:53:18.476 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:53:18 smithi136 bash[68240]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4 2024-09-17T09:53:18.476 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:18 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:18.241+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.814583+0000 front 2024-09-17T09:52:44.814673+0000 (oldest deadline 2024-09-17T09:53:08.314503+0000) 2024-09-17T09:53:18.476 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:18 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:18.241+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:51.116044+0000 front 2024-09-17T09:52:51.116104+0000 (oldest deadline 2024-09-17T09:53:17.015934+0000) 2024-09-17T09:53:18.476 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:18 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:18.196+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:18.476 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:18 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:18.196+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.810464+0000 front 2024-09-17T09:52:50.810449+0000 (oldest deadline 2024-09-17T09:53:14.910321+0000) 2024-09-17T09:53:18.975 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:53:18 smithi136 podman[68256]: 2024-09-17 09:53:18.675115408 +0000 UTC m=+0.762629300 container remove e2a18ad9466438e628af3d470891ea0615e10c803666172b228adffff0ecee82 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-17T09:53:18.976 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:18 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:18.820+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:40.904179+0000 front 2024-09-17T09:52:40.904227+0000 (oldest deadline 2024-09-17T09:53:06.204000+0000) 2024-09-17T09:53:18.976 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:18 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:18.820+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.205516+0000 front 2024-09-17T09:52:50.205563+0000 (oldest deadline 2024-09-17T09:53:14.305408+0000) 2024-09-17T09:53:19.284 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:19 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:19.236+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:19.284 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:19 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:19.236+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.810464+0000 front 2024-09-17T09:52:50.810449+0000 (oldest deadline 2024-09-17T09:53:14.910321+0000) 2024-09-17T09:53:19.285 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:53:18 smithi136 podman[68342]: 2024-09-17 09:53:18.894773811 +0000 UTC m=+0.015617828 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:53:19.285 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:53:19 smithi136 podman[68342]: 2024-09-17 09:53:19.009821437 +0000 UTC m=+0.130665484 container create 40bb47aad641a9ec2001c1a1d8caaf11fad7c66e8aa711e0109418a4bffe6d6f (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.build-date=20240731, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T09:53:19.285 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:53:19 smithi136 podman[68342]: 2024-09-17 09:53:19.228842837 +0000 UTC m=+0.349686853 container init 40bb47aad641a9ec2001c1a1d8caaf11fad7c66e8aa711e0109418a4bffe6d6f (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72) 2024-09-17T09:53:19.285 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:53:19 smithi136 podman[68342]: 2024-09-17 09:53:19.232016037 +0000 UTC m=+0.352860045 container start 40bb47aad641a9ec2001c1a1d8caaf11fad7c66e8aa711e0109418a4bffe6d6f (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4-deactivate, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-17T09:53:19.726 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:19 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:19.282+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.814583+0000 front 2024-09-17T09:52:44.814673+0000 (oldest deadline 2024-09-17T09:53:08.314503+0000) 2024-09-17T09:53:19.726 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:19 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:19.282+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:51.116044+0000 front 2024-09-17T09:52:51.116104+0000 (oldest deadline 2024-09-17T09:53:17.015934+0000) 2024-09-17T09:53:19.726 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:53:19 smithi136 podman[68342]: 2024-09-17 09:53:19.337377806 +0000 UTC m=+0.458221821 container attach 40bb47aad641a9ec2001c1a1d8caaf11fad7c66e8aa711e0109418a4bffe6d6f (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T09:53:19.726 INFO:journalctl@ceph.osd.4.smithi136.stdout:Sep 17 09:53:19 smithi136 podman[68342]: 2024-09-17 09:53:19.422315626 +0000 UTC m=+0.543159637 container died 40bb47aad641a9ec2001c1a1d8caaf11fad7c66e8aa711e0109418a4bffe6d6f (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-4-deactivate, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-17T09:53:20.142 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:19 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:19.820+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:40.904179+0000 front 2024-09-17T09:52:40.904227+0000 (oldest deadline 2024-09-17T09:53:06.204000+0000) 2024-09-17T09:53:20.142 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:19 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:19.820+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.205516+0000 front 2024-09-17T09:52:50.205563+0000 (oldest deadline 2024-09-17T09:53:14.305408+0000) 2024-09-17T09:53:20.163 DEBUG:teuthology.orchestra.run.smithi136:> sudo pkill -f 'journalctl -f -n 0 -u ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.4.service' 2024-09-17T09:53:20.203 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T09:53:20.203 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-17T09:53:20.203 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-17T09:53:20.203 DEBUG:teuthology.orchestra.run.smithi136:> sudo systemctl stop ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.5 2024-09-17T09:53:20.475 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:20 smithi136 systemd[1]: Stopping Ceph osd.5 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:53:20.476 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:20 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:20.302+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.814583+0000 front 2024-09-17T09:52:44.814673+0000 (oldest deadline 2024-09-17T09:53:08.314503+0000) 2024-09-17T09:53:20.476 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:20 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:20.302+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:51.116044+0000 front 2024-09-17T09:52:51.116104+0000 (oldest deadline 2024-09-17T09:53:17.015934+0000) 2024-09-17T09:53:20.476 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:20 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:20.226+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:20.476 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:20 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:20.226+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.810464+0000 front 2024-09-17T09:52:50.810449+0000 (oldest deadline 2024-09-17T09:53:14.910321+0000) 2024-09-17T09:53:20.976 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:20 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:20.536+0000 7f90945f2640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:53:20.976 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:20 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:20.536+0000 7f90945f2640 -1 osd.5 161 *** Got signal Terminated *** 2024-09-17T09:53:20.976 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:20 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:20.536+0000 7f90945f2640 -1 osd.5 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:53:20.976 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:20 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:20.784+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:40.904179+0000 front 2024-09-17T09:52:40.904227+0000 (oldest deadline 2024-09-17T09:53:06.204000+0000) 2024-09-17T09:53:20.976 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:20 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:20.784+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.205516+0000 front 2024-09-17T09:52:50.205563+0000 (oldest deadline 2024-09-17T09:53:14.305408+0000) 2024-09-17T09:53:21.725 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:21 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:21.255+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.814583+0000 front 2024-09-17T09:52:44.814673+0000 (oldest deadline 2024-09-17T09:53:08.314503+0000) 2024-09-17T09:53:21.726 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:21 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:21.255+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:51.116044+0000 front 2024-09-17T09:52:51.116104+0000 (oldest deadline 2024-09-17T09:53:17.015934+0000) 2024-09-17T09:53:21.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:21 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:21.237+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:21.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:21 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:21.237+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.810464+0000 front 2024-09-17T09:52:50.810449+0000 (oldest deadline 2024-09-17T09:53:14.910321+0000) 2024-09-17T09:53:22.225 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:21 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:21.799+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:40.904179+0000 front 2024-09-17T09:52:40.904227+0000 (oldest deadline 2024-09-17T09:53:06.204000+0000) 2024-09-17T09:53:22.226 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:21 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:21.799+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.205516+0000 front 2024-09-17T09:52:50.205563+0000 (oldest deadline 2024-09-17T09:53:14.305408+0000) 2024-09-17T09:53:22.725 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:22 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:22.281+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.814583+0000 front 2024-09-17T09:52:44.814673+0000 (oldest deadline 2024-09-17T09:53:08.314503+0000) 2024-09-17T09:53:22.726 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:22 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:22.281+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:51.116044+0000 front 2024-09-17T09:52:51.116104+0000 (oldest deadline 2024-09-17T09:53:17.015934+0000) 2024-09-17T09:53:22.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:22 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:22.254+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:22.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:22 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:22.254+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.810464+0000 front 2024-09-17T09:52:50.810449+0000 (oldest deadline 2024-09-17T09:53:14.910321+0000) 2024-09-17T09:53:23.225 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:22 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:22.845+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:40.904179+0000 front 2024-09-17T09:52:40.904227+0000 (oldest deadline 2024-09-17T09:53:06.204000+0000) 2024-09-17T09:53:23.226 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:22 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:22.845+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.205516+0000 front 2024-09-17T09:52:50.205563+0000 (oldest deadline 2024-09-17T09:53:14.305408+0000) 2024-09-17T09:53:23.226 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:22 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:22.845+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:58.906623+0000 front 2024-09-17T09:52:58.906722+0000 (oldest deadline 2024-09-17T09:53:22.406514+0000) 2024-09-17T09:53:23.725 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:23 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:23.270+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.814583+0000 front 2024-09-17T09:52:44.814673+0000 (oldest deadline 2024-09-17T09:53:08.314503+0000) 2024-09-17T09:53:23.726 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:23 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:23.270+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:51.116044+0000 front 2024-09-17T09:52:51.116104+0000 (oldest deadline 2024-09-17T09:53:17.015934+0000) 2024-09-17T09:53:23.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:23 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:23.303+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:23.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:23 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:23.303+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.810464+0000 front 2024-09-17T09:52:50.810449+0000 (oldest deadline 2024-09-17T09:53:14.910321+0000) 2024-09-17T09:53:23.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:23 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:23.303+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:59.011162+0000 front 2024-09-17T09:52:59.011148+0000 (oldest deadline 2024-09-17T09:53:23.110997+0000) 2024-09-17T09:53:24.226 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:23 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:23.843+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:40.904179+0000 front 2024-09-17T09:52:40.904227+0000 (oldest deadline 2024-09-17T09:53:06.204000+0000) 2024-09-17T09:53:24.226 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:23 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:23.843+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.205516+0000 front 2024-09-17T09:52:50.205563+0000 (oldest deadline 2024-09-17T09:53:14.305408+0000) 2024-09-17T09:53:24.226 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:23 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:23.843+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:58.906623+0000 front 2024-09-17T09:52:58.906722+0000 (oldest deadline 2024-09-17T09:53:22.406514+0000) 2024-09-17T09:53:24.725 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:24 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:24.266+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.814583+0000 front 2024-09-17T09:52:44.814673+0000 (oldest deadline 2024-09-17T09:53:08.314503+0000) 2024-09-17T09:53:24.726 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:24 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:24.266+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:51.116044+0000 front 2024-09-17T09:52:51.116104+0000 (oldest deadline 2024-09-17T09:53:17.015934+0000) 2024-09-17T09:53:24.726 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:24 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:24.266+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:59.316869+0000 front 2024-09-17T09:52:59.316894+0000 (oldest deadline 2024-09-17T09:53:23.416691+0000) 2024-09-17T09:53:24.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:24 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:24.320+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:24.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:24 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:24.320+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.810464+0000 front 2024-09-17T09:52:50.810449+0000 (oldest deadline 2024-09-17T09:53:14.910321+0000) 2024-09-17T09:53:24.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:24 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:24.320+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:59.011162+0000 front 2024-09-17T09:52:59.011148+0000 (oldest deadline 2024-09-17T09:53:23.110997+0000) 2024-09-17T09:53:25.225 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:24 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:24.808+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:40.904179+0000 front 2024-09-17T09:52:40.904227+0000 (oldest deadline 2024-09-17T09:53:06.204000+0000) 2024-09-17T09:53:25.226 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:24 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:24.808+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.205516+0000 front 2024-09-17T09:52:50.205563+0000 (oldest deadline 2024-09-17T09:53:14.305408+0000) 2024-09-17T09:53:25.226 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:24 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5[55426]: 2024-09-17T09:53:24.808+0000 7f908cbff640 -1 osd.5 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:58.906623+0000 front 2024-09-17T09:52:58.906722+0000 (oldest deadline 2024-09-17T09:53:22.406514+0000) 2024-09-17T09:53:25.726 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:25 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:25.271+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.814583+0000 front 2024-09-17T09:52:44.814673+0000 (oldest deadline 2024-09-17T09:53:08.314503+0000) 2024-09-17T09:53:25.726 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:25 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:25.271+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:51.116044+0000 front 2024-09-17T09:52:51.116104+0000 (oldest deadline 2024-09-17T09:53:17.015934+0000) 2024-09-17T09:53:25.726 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:25 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:25.271+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:59.316869+0000 front 2024-09-17T09:52:59.316894+0000 (oldest deadline 2024-09-17T09:53:23.416691+0000) 2024-09-17T09:53:25.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:25 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:25.295+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:25.727 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:25 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:25.295+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.810464+0000 front 2024-09-17T09:52:50.810449+0000 (oldest deadline 2024-09-17T09:53:14.910321+0000) 2024-09-17T09:53:25.727 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:25 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:25.295+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:59.011162+0000 front 2024-09-17T09:52:59.011148+0000 (oldest deadline 2024-09-17T09:53:23.110997+0000) 2024-09-17T09:53:26.271 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:25 smithi136 podman[68446]: 2024-09-17 09:53:25.980699711 +0000 UTC m=+5.555577913 container died df9b6954ce0f7365eb4b5ee5f2940b7c9bc6589537b8702945231786d886afbc (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T09:53:26.722 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:26 smithi136 podman[68446]: 2024-09-17 09:53:26.444639522 +0000 UTC m=+6.019517755 container cleanup df9b6954ce0f7365eb4b5ee5f2940b7c9bc6589537b8702945231786d886afbc (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-17T09:53:26.722 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:26 smithi136 bash[68446]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5 2024-09-17T09:53:26.723 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:26 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:26.321+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.814583+0000 front 2024-09-17T09:52:44.814673+0000 (oldest deadline 2024-09-17T09:53:08.314503+0000) 2024-09-17T09:53:26.723 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:26 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:26.321+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:51.116044+0000 front 2024-09-17T09:52:51.116104+0000 (oldest deadline 2024-09-17T09:53:17.015934+0000) 2024-09-17T09:53:26.723 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:26 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:26.321+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:59.316869+0000 front 2024-09-17T09:52:59.316894+0000 (oldest deadline 2024-09-17T09:53:23.416691+0000) 2024-09-17T09:53:26.723 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:26 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:26.290+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:26.723 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:26 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:26.290+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.810464+0000 front 2024-09-17T09:52:50.810449+0000 (oldest deadline 2024-09-17T09:53:14.910321+0000) 2024-09-17T09:53:26.723 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:26 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:26.290+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:59.011162+0000 front 2024-09-17T09:52:59.011148+0000 (oldest deadline 2024-09-17T09:53:23.110997+0000) 2024-09-17T09:53:26.975 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:26 smithi136 podman[68478]: 2024-09-17 09:53:26.721125562 +0000 UTC m=+0.739769010 container remove df9b6954ce0f7365eb4b5ee5f2940b7c9bc6589537b8702945231786d886afbc (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-17T09:53:27.317 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:27 smithi136 podman[68569]: 2024-09-17 09:53:26.961707357 +0000 UTC m=+0.015720023 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:53:27.317 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:27 smithi136 podman[68569]: 2024-09-17 09:53:27.080491075 +0000 UTC m=+0.134503742 container create ac50eb06937805aa92f06a98c8b60cf6965eb7e5d16217fccfb64ec9aadec10b (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5-deactivate, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T09:53:27.317 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:27 smithi136 podman[68569]: 2024-09-17 09:53:27.316240212 +0000 UTC m=+0.370252891 container init ac50eb06937805aa92f06a98c8b60cf6965eb7e5d16217fccfb64ec9aadec10b (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS) 2024-09-17T09:53:27.725 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:27 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:27.323+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.814583+0000 front 2024-09-17T09:52:44.814673+0000 (oldest deadline 2024-09-17T09:53:08.314503+0000) 2024-09-17T09:53:27.726 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:27 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:27.323+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:51.116044+0000 front 2024-09-17T09:52:51.116104+0000 (oldest deadline 2024-09-17T09:53:17.015934+0000) 2024-09-17T09:53:27.726 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:27 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:27.323+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:59.316869+0000 front 2024-09-17T09:52:59.316894+0000 (oldest deadline 2024-09-17T09:53:23.416691+0000) 2024-09-17T09:53:27.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:27 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:27.332+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:27.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:27 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:27.332+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.810464+0000 front 2024-09-17T09:52:50.810449+0000 (oldest deadline 2024-09-17T09:53:14.910321+0000) 2024-09-17T09:53:27.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:27 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:27.332+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:59.011162+0000 front 2024-09-17T09:52:59.011148+0000 (oldest deadline 2024-09-17T09:53:23.110997+0000) 2024-09-17T09:53:27.726 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:27 smithi136 podman[68569]: 2024-09-17 09:53:27.319492391 +0000 UTC m=+0.373505065 container start ac50eb06937805aa92f06a98c8b60cf6965eb7e5d16217fccfb64ec9aadec10b (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-17T09:53:27.727 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:27 smithi136 podman[68569]: 2024-09-17 09:53:27.433317569 +0000 UTC m=+0.487330244 container attach ac50eb06937805aa92f06a98c8b60cf6965eb7e5d16217fccfb64ec9aadec10b (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.schema-version=1.0) 2024-09-17T09:53:27.727 INFO:journalctl@ceph.osd.5.smithi136.stdout:Sep 17 09:53:27 smithi136 podman[68569]: 2024-09-17 09:53:27.504175092 +0000 UTC m=+0.558187758 container died ac50eb06937805aa92f06a98c8b60cf6965eb7e5d16217fccfb64ec9aadec10b (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-17T09:53:28.166 DEBUG:teuthology.orchestra.run.smithi136:> sudo pkill -f 'journalctl -f -n 0 -u ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.5.service' 2024-09-17T09:53:28.204 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T09:53:28.204 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-17T09:53:28.205 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-17T09:53:28.205 DEBUG:teuthology.orchestra.run.smithi136:> sudo systemctl stop ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.6 2024-09-17T09:53:28.476 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:28 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:28.341+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:28.476 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:28 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:28.341+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.810464+0000 front 2024-09-17T09:52:50.810449+0000 (oldest deadline 2024-09-17T09:53:14.910321+0000) 2024-09-17T09:53:28.476 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:28 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:28.341+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:59.011162+0000 front 2024-09-17T09:52:59.011148+0000 (oldest deadline 2024-09-17T09:53:23.110997+0000) 2024-09-17T09:53:28.477 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:28 smithi136 systemd[1]: Stopping Ceph osd.6 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:53:28.477 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:28 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:28.327+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.814583+0000 front 2024-09-17T09:52:44.814673+0000 (oldest deadline 2024-09-17T09:53:08.314503+0000) 2024-09-17T09:53:28.477 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:28 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:28.327+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:51.116044+0000 front 2024-09-17T09:52:51.116104+0000 (oldest deadline 2024-09-17T09:53:17.015934+0000) 2024-09-17T09:53:28.477 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:28 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:28.327+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:59.316869+0000 front 2024-09-17T09:52:59.316894+0000 (oldest deadline 2024-09-17T09:53:23.416691+0000) 2024-09-17T09:53:28.975 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:28 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:28.531+0000 7f07a5854640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:53:28.976 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:28 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:28.531+0000 7f07a5854640 -1 osd.6 161 *** Got signal Terminated *** 2024-09-17T09:53:28.976 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:28 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:28.531+0000 7f07a5854640 -1 osd.6 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:53:29.726 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:29 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:29.306+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.814583+0000 front 2024-09-17T09:52:44.814673+0000 (oldest deadline 2024-09-17T09:53:08.314503+0000) 2024-09-17T09:53:29.726 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:29 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:29.306+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:51.116044+0000 front 2024-09-17T09:52:51.116104+0000 (oldest deadline 2024-09-17T09:53:17.015934+0000) 2024-09-17T09:53:29.726 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:29 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:29.306+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:59.316869+0000 front 2024-09-17T09:52:59.316894+0000 (oldest deadline 2024-09-17T09:53:23.416691+0000) 2024-09-17T09:53:29.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:29 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:29.389+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:29.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:29 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:29.389+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.810464+0000 front 2024-09-17T09:52:50.810449+0000 (oldest deadline 2024-09-17T09:53:14.910321+0000) 2024-09-17T09:53:29.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:29 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:29.389+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:59.011162+0000 front 2024-09-17T09:52:59.011148+0000 (oldest deadline 2024-09-17T09:53:23.110997+0000) 2024-09-17T09:53:30.726 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:30 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:30.258+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.814583+0000 front 2024-09-17T09:52:44.814673+0000 (oldest deadline 2024-09-17T09:53:08.314503+0000) 2024-09-17T09:53:30.761 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:30 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:30.258+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:51.116044+0000 front 2024-09-17T09:52:51.116104+0000 (oldest deadline 2024-09-17T09:53:17.015934+0000) 2024-09-17T09:53:30.762 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:30 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:30.258+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:59.316869+0000 front 2024-09-17T09:52:59.316894+0000 (oldest deadline 2024-09-17T09:53:23.416691+0000) 2024-09-17T09:53:30.762 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:30 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:30.373+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:30.762 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:30 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:30.373+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.810464+0000 front 2024-09-17T09:52:50.810449+0000 (oldest deadline 2024-09-17T09:53:14.910321+0000) 2024-09-17T09:53:30.762 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:30 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:30.373+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:59.011162+0000 front 2024-09-17T09:52:59.011148+0000 (oldest deadline 2024-09-17T09:53:23.110997+0000) 2024-09-17T09:53:31.725 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:31 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:31.229+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.814583+0000 front 2024-09-17T09:52:44.814673+0000 (oldest deadline 2024-09-17T09:53:08.314503+0000) 2024-09-17T09:53:31.726 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:31 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:31.229+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:51.116044+0000 front 2024-09-17T09:52:51.116104+0000 (oldest deadline 2024-09-17T09:53:17.015934+0000) 2024-09-17T09:53:31.726 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:31 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:31.229+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:59.316869+0000 front 2024-09-17T09:52:59.316894+0000 (oldest deadline 2024-09-17T09:53:23.416691+0000) 2024-09-17T09:53:31.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:31 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:31.337+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:31.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:31 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:31.337+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.810464+0000 front 2024-09-17T09:52:50.810449+0000 (oldest deadline 2024-09-17T09:53:14.910321+0000) 2024-09-17T09:53:31.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:31 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:31.337+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:59.011162+0000 front 2024-09-17T09:52:59.011148+0000 (oldest deadline 2024-09-17T09:53:23.110997+0000) 2024-09-17T09:53:31.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:31 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:31.337+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-17T09:53:08.812605+0000 front 2024-09-17T09:53:08.812591+0000 (oldest deadline 2024-09-17T09:53:30.512487+0000) 2024-09-17T09:53:32.476 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:32 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:32.214+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.814583+0000 front 2024-09-17T09:52:44.814673+0000 (oldest deadline 2024-09-17T09:53:08.314503+0000) 2024-09-17T09:53:32.476 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:32 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:32.214+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:51.116044+0000 front 2024-09-17T09:52:51.116104+0000 (oldest deadline 2024-09-17T09:53:17.015934+0000) 2024-09-17T09:53:32.476 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:32 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:32.214+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:59.316869+0000 front 2024-09-17T09:52:59.316894+0000 (oldest deadline 2024-09-17T09:53:23.416691+0000) 2024-09-17T09:53:32.477 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:32 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:32.344+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:32.477 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:32 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:32.344+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.810464+0000 front 2024-09-17T09:52:50.810449+0000 (oldest deadline 2024-09-17T09:53:14.910321+0000) 2024-09-17T09:53:32.477 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:32 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:32.344+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:59.011162+0000 front 2024-09-17T09:52:59.011148+0000 (oldest deadline 2024-09-17T09:53:23.110997+0000) 2024-09-17T09:53:32.477 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:32 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:32.344+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-17T09:53:08.812605+0000 front 2024-09-17T09:53:08.812591+0000 (oldest deadline 2024-09-17T09:53:30.512487+0000) 2024-09-17T09:53:33.726 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:33 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:33.234+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.814583+0000 front 2024-09-17T09:52:44.814673+0000 (oldest deadline 2024-09-17T09:53:08.314503+0000) 2024-09-17T09:53:33.726 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:33 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:33.234+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:51.116044+0000 front 2024-09-17T09:52:51.116104+0000 (oldest deadline 2024-09-17T09:53:17.015934+0000) 2024-09-17T09:53:33.726 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:33 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6[57471]: 2024-09-17T09:53:33.234+0000 7f079de61640 -1 osd.6 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:59.316869+0000 front 2024-09-17T09:52:59.316894+0000 (oldest deadline 2024-09-17T09:53:23.416691+0000) 2024-09-17T09:53:33.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:33 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:33.381+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:33.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:33 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:33.381+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.810464+0000 front 2024-09-17T09:52:50.810449+0000 (oldest deadline 2024-09-17T09:53:14.910321+0000) 2024-09-17T09:53:33.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:33 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:33.381+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:59.011162+0000 front 2024-09-17T09:52:59.011148+0000 (oldest deadline 2024-09-17T09:53:23.110997+0000) 2024-09-17T09:53:33.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:33 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:33.381+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-17T09:53:08.812605+0000 front 2024-09-17T09:53:08.812591+0000 (oldest deadline 2024-09-17T09:53:30.512487+0000) 2024-09-17T09:53:34.199 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:33 smithi136 podman[68671]: 2024-09-17 09:53:33.936444265 +0000 UTC m=+5.511929997 container died c477e7043301df66181a9bda379471b3c3b49a4e4890512fe62633f6595891ad (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-17T09:53:34.476 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:34 smithi136 podman[68671]: 2024-09-17 09:53:34.372517739 +0000 UTC m=+5.948003469 container cleanup c477e7043301df66181a9bda379471b3c3b49a4e4890512fe62633f6595891ad (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-17T09:53:34.476 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:34 smithi136 bash[68671]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6 2024-09-17T09:53:34.476 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:34 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:34.354+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:34.476 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:34 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:34.354+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.810464+0000 front 2024-09-17T09:52:50.810449+0000 (oldest deadline 2024-09-17T09:53:14.910321+0000) 2024-09-17T09:53:34.476 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:34 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:34.354+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:59.011162+0000 front 2024-09-17T09:52:59.011148+0000 (oldest deadline 2024-09-17T09:53:23.110997+0000) 2024-09-17T09:53:34.477 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:34 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:34.354+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-17T09:53:08.812605+0000 front 2024-09-17T09:53:08.812591+0000 (oldest deadline 2024-09-17T09:53:30.512487+0000) 2024-09-17T09:53:34.976 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:34 smithi136 podman[68688]: 2024-09-17 09:53:34.58255586 +0000 UTC m=+0.642893766 container remove c477e7043301df66181a9bda379471b3c3b49a4e4890512fe62633f6595891ad (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T09:53:35.320 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:34 smithi136 podman[68777]: 2024-09-17 09:53:34.883596603 +0000 UTC m=+0.015863680 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:53:35.320 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:35 smithi136 podman[68777]: 2024-09-17 09:53:35.016790673 +0000 UTC m=+0.149057733 container create 04b94bf07847f2828a4c03bfd25992298d4ec7b3072ca03d4b87ea6ffc7ca0e2 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6-deactivate, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T09:53:35.320 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:35 smithi136 podman[68777]: 2024-09-17 09:53:35.227254352 +0000 UTC m=+0.359521420 container init 04b94bf07847f2828a4c03bfd25992298d4ec7b3072ca03d4b87ea6ffc7ca0e2 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-17T09:53:35.320 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:35 smithi136 podman[68777]: 2024-09-17 09:53:35.230756587 +0000 UTC m=+0.363023647 container start 04b94bf07847f2828a4c03bfd25992298d4ec7b3072ca03d4b87ea6ffc7ca0e2 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, ceph=True, org.label-schema.vendor=CentOS) 2024-09-17T09:53:35.320 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:35 smithi136 podman[68777]: 2024-09-17 09:53:35.319017369 +0000 UTC m=+0.451284431 container attach 04b94bf07847f2828a4c03bfd25992298d4ec7b3072ca03d4b87ea6ffc7ca0e2 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-17T09:53:35.725 INFO:journalctl@ceph.osd.6.smithi136.stdout:Sep 17 09:53:35 smithi136 podman[68777]: 2024-09-17 09:53:35.41951508 +0000 UTC m=+0.551782140 container died 04b94bf07847f2828a4c03bfd25992298d4ec7b3072ca03d4b87ea6ffc7ca0e2 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-6-deactivate, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T09:53:35.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:35 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:35.384+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:35.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:35 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:35.384+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.810464+0000 front 2024-09-17T09:52:50.810449+0000 (oldest deadline 2024-09-17T09:53:14.910321+0000) 2024-09-17T09:53:35.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:35 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:35.384+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:59.011162+0000 front 2024-09-17T09:52:59.011148+0000 (oldest deadline 2024-09-17T09:53:23.110997+0000) 2024-09-17T09:53:35.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:35 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:35.384+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-17T09:53:08.812605+0000 front 2024-09-17T09:53:08.812591+0000 (oldest deadline 2024-09-17T09:53:30.512487+0000) 2024-09-17T09:53:36.280 DEBUG:teuthology.orchestra.run.smithi136:> sudo pkill -f 'journalctl -f -n 0 -u ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.6.service' 2024-09-17T09:53:36.321 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T09:53:36.321 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-17T09:53:36.321 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-17T09:53:36.321 DEBUG:teuthology.orchestra.run.smithi136:> sudo systemctl stop ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.7 2024-09-17T09:53:36.649 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:36 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:36.347+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:36.649 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:36 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:36.347+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.810464+0000 front 2024-09-17T09:52:50.810449+0000 (oldest deadline 2024-09-17T09:53:14.910321+0000) 2024-09-17T09:53:36.650 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:36 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:36.347+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:59.011162+0000 front 2024-09-17T09:52:59.011148+0000 (oldest deadline 2024-09-17T09:53:23.110997+0000) 2024-09-17T09:53:36.650 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:36 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:36.347+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-17T09:53:08.812605+0000 front 2024-09-17T09:53:08.812591+0000 (oldest deadline 2024-09-17T09:53:30.512487+0000) 2024-09-17T09:53:36.650 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:36 smithi136 systemd[1]: Stopping Ceph osd.7 for 35c0cd22-74d7-11ef-bceb-c7b262605968... 2024-09-17T09:53:36.650 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:36 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:36.647+0000 7ffa9da78640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T09:53:36.650 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:36 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:36.647+0000 7ffa9da78640 -1 osd.7 161 *** Got signal Terminated *** 2024-09-17T09:53:36.650 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:36 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:36.647+0000 7ffa9da78640 -1 osd.7 161 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T09:53:37.725 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:37 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:37.330+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:37.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:37 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:37.330+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.810464+0000 front 2024-09-17T09:52:50.810449+0000 (oldest deadline 2024-09-17T09:53:14.910321+0000) 2024-09-17T09:53:37.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:37 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:37.330+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:59.011162+0000 front 2024-09-17T09:52:59.011148+0000 (oldest deadline 2024-09-17T09:53:23.110997+0000) 2024-09-17T09:53:37.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:37 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:37.330+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-17T09:53:08.812605+0000 front 2024-09-17T09:53:08.812591+0000 (oldest deadline 2024-09-17T09:53:30.512487+0000) 2024-09-17T09:53:38.476 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:38 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:38.319+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:38.476 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:38 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:38.319+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.810464+0000 front 2024-09-17T09:52:50.810449+0000 (oldest deadline 2024-09-17T09:53:14.910321+0000) 2024-09-17T09:53:38.476 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:38 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:38.319+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:59.011162+0000 front 2024-09-17T09:52:59.011148+0000 (oldest deadline 2024-09-17T09:53:23.110997+0000) 2024-09-17T09:53:38.476 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:38 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:38.319+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-17T09:53:08.812605+0000 front 2024-09-17T09:53:08.812591+0000 (oldest deadline 2024-09-17T09:53:30.512487+0000) 2024-09-17T09:53:39.725 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:39 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:39.362+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:39.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:39 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:39.363+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.810464+0000 front 2024-09-17T09:52:50.810449+0000 (oldest deadline 2024-09-17T09:53:14.910321+0000) 2024-09-17T09:53:39.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:39 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:39.363+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:59.011162+0000 front 2024-09-17T09:52:59.011148+0000 (oldest deadline 2024-09-17T09:53:23.110997+0000) 2024-09-17T09:53:39.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:39 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:39.363+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-17T09:53:08.812605+0000 front 2024-09-17T09:53:08.812591+0000 (oldest deadline 2024-09-17T09:53:30.512487+0000) 2024-09-17T09:53:40.725 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:40 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:40.377+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:40.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:40 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:40.377+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.810464+0000 front 2024-09-17T09:52:50.810449+0000 (oldest deadline 2024-09-17T09:53:14.910321+0000) 2024-09-17T09:53:40.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:40 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:40.377+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:59.011162+0000 front 2024-09-17T09:52:59.011148+0000 (oldest deadline 2024-09-17T09:53:23.110997+0000) 2024-09-17T09:53:40.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:40 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:40.377+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-17T09:53:08.812605+0000 front 2024-09-17T09:53:08.812591+0000 (oldest deadline 2024-09-17T09:53:30.512487+0000) 2024-09-17T09:53:40.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:40 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:40.377+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.136:6804 osd.4 since back 2024-09-17T09:53:14.013231+0000 front 2024-09-17T09:53:14.013280+0000 (oldest deadline 2024-09-17T09:53:39.913259+0000) 2024-09-17T09:53:41.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:41 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:41.331+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-17T09:52:44.409796+0000 front 2024-09-17T09:52:44.409714+0000 (oldest deadline 2024-09-17T09:53:07.909629+0000) 2024-09-17T09:53:41.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:41 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:41.331+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-17T09:52:50.810464+0000 front 2024-09-17T09:52:50.810449+0000 (oldest deadline 2024-09-17T09:53:14.910321+0000) 2024-09-17T09:53:41.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:41 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:41.331+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-17T09:52:59.011162+0000 front 2024-09-17T09:52:59.011148+0000 (oldest deadline 2024-09-17T09:53:23.110997+0000) 2024-09-17T09:53:41.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:41 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:41.331+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-17T09:53:08.812605+0000 front 2024-09-17T09:53:08.812591+0000 (oldest deadline 2024-09-17T09:53:30.512487+0000) 2024-09-17T09:53:41.726 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:41 smithi136 ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7[59496]: 2024-09-17T09:53:41.331+0000 7ffa9a08d640 -1 osd.7 161 heartbeat_check: no reply from 172.21.15.136:6804 osd.4 since back 2024-09-17T09:53:14.013231+0000 front 2024-09-17T09:53:14.013280+0000 (oldest deadline 2024-09-17T09:53:39.913259+0000) 2024-09-17T09:53:42.475 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:42 smithi136 podman[68882]: 2024-09-17 09:53:42.221635938 +0000 UTC m=+5.678582040 container died 6c9da904d39c62fe16c5011380e5067007db82e889f0a654e23f5cdaec5ebb99 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-17T09:53:42.976 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:42 smithi136 podman[68882]: 2024-09-17 09:53:42.686454791 +0000 UTC m=+6.143400881 container cleanup 6c9da904d39c62fe16c5011380e5067007db82e889f0a654e23f5cdaec5ebb99 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-17T09:53:42.977 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:42 smithi136 bash[68882]: ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7 2024-09-17T09:53:43.225 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:42 smithi136 podman[68909]: 2024-09-17 09:53:42.974259519 +0000 UTC m=+0.750646280 container remove 6c9da904d39c62fe16c5011380e5067007db82e889f0a654e23f5cdaec5ebb99 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-17T09:53:43.628 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:43 smithi136 podman[68998]: 2024-09-17 09:53:43.200965363 +0000 UTC m=+0.016182542 image pull f271290a09505c16fdd5580c0844868087b89a7f4f5340b8b5900ad1017d802b quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407 2024-09-17T09:53:43.628 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:43 smithi136 podman[68998]: 2024-09-17 09:53:43.325114121 +0000 UTC m=+0.140331279 container create a621dccc9e448ac3116e5bb43508128fc871815deddb9f8ff8ae17cf1946d490 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-17T09:53:43.628 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:43 smithi136 podman[68998]: 2024-09-17 09:53:43.54419986 +0000 UTC m=+0.359417038 container init a621dccc9e448ac3116e5bb43508128fc871815deddb9f8ff8ae17cf1946d490 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-17T09:53:43.628 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:43 smithi136 podman[68998]: 2024-09-17 09:53:43.547585758 +0000 UTC m=+0.362802921 container start a621dccc9e448ac3116e5bb43508128fc871815deddb9f8ff8ae17cf1946d490 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T09:53:43.629 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:43 smithi136 podman[68998]: 2024-09-17 09:53:43.627422962 +0000 UTC m=+0.442640122 container attach a621dccc9e448ac3116e5bb43508128fc871815deddb9f8ff8ae17cf1946d490 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, maintainer=Guillaume Abrioux ) 2024-09-17T09:53:43.976 INFO:journalctl@ceph.osd.7.smithi136.stdout:Sep 17 09:53:43 smithi136 podman[68998]: 2024-09-17 09:53:43.73839782 +0000 UTC m=+0.553614989 container died a621dccc9e448ac3116e5bb43508128fc871815deddb9f8ff8ae17cf1946d490 (image=quay.ceph.io/ceph-ci/ceph@sha256:e744bf8cae79f6e0ef98cbe9e42368cf98408f8bd0d4b2795862ed11d9d12407, name=ceph-35c0cd22-74d7-11ef-bceb-c7b262605968-osd-7-deactivate, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-athakkar-testing-2024-09-17-0048-e646f72, ceph=True) 2024-09-17T09:53:44.385 DEBUG:teuthology.orchestra.run.smithi136:> sudo pkill -f 'journalctl -f -n 0 -u ceph-35c0cd22-74d7-11ef-bceb-c7b262605968@osd.7.service' 2024-09-17T09:53:44.422 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T09:53:44.422 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-17T09:53:44.423 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-17T09:56:41.226 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-17T09:57:02.879 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-17T09:57:02.910 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-17T09:57:02.937 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-17T09:57:02.939 DEBUG:teuthology.misc:Transferring archived files from smithi078:/var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/avan-2024-09-17_09:09:13-orch:cephadm-wip-athakkar-testing-2024-09-17-0048-distro-default-smithi/7908750/remote/smithi078/crash 2024-09-17T09:57:02.940 DEBUG:teuthology.orchestra.run.smithi078:> sudo tar c -f - -C /var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/crash -- . 2024-09-17T09:57:02.999 INFO:teuthology.orchestra.run.smithi078.stderr:tar: /var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-17T09:57:02.999 INFO:teuthology.orchestra.run.smithi078.stderr:tar: Error is not recoverable: exiting now 2024-09-17T09:57:03.001 DEBUG:teuthology.misc:Transferring archived files from smithi136:/var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/avan-2024-09-17_09:09:13-orch:cephadm-wip-athakkar-testing-2024-09-17-0048-distro-default-smithi/7908750/remote/smithi136/crash 2024-09-17T09:57:03.002 DEBUG:teuthology.orchestra.run.smithi136:> sudo tar c -f - -C /var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/crash -- . 2024-09-17T09:57:03.039 INFO:teuthology.orchestra.run.smithi136.stderr:tar: /var/lib/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-17T09:57:03.039 INFO:teuthology.orchestra.run.smithi136.stderr:tar: Error is not recoverable: exiting now 2024-09-17T09:57:03.040 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-17T09:57:03.040 DEBUG:teuthology.orchestra.run.smithi078:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v CEPHADM_REFRESH_FAILED | head -n 1 2024-09-17T09:57:03.069 INFO:tasks.cephadm:Compressing logs... 2024-09-17T09:57:03.069 DEBUG:teuthology.orchestra.run.smithi078:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-17T09:57:03.112 DEBUG:teuthology.orchestra.run.smithi136:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-17T09:57:03.134 INFO:teuthology.orchestra.run.smithi078.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-osd.0.log 2024-09-17T09:57:03.134 INFO:teuthology.orchestra.run.smithi078.stderr:: No such file or directory 2024-09-17T09:57:03.134 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-osd.1.log 2024-09-17T09:57:03.135 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-17T09:57:03.135 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-mon.smithi078.log 2024-09-17T09:57:03.135 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-17T09:57:03.141 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-mon.smithi078.log: gzip -5 --verbose -- /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-17T09:57:03.141 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-mgr.smithi078.rsqofi.log 2024-09-17T09:57:03.141 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi078.jdrsup.log 2024-09-17T09:57:03.141 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-mgr.smithi078.rsqofi.log: gzip -5 --verbose -- /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-osd.2.log 2024-09-17T09:57:03.141 INFO:teuthology.orchestra.run.smithi078.stderr: 85.8% -- replaced with /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-17T09:57:03.141 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi078.jdrsup.log: gzip -5 --verbose -- /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph.log 2024-09-17T09:57:03.141 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-osd.2.log: 89.2% -- replaced with /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi078.jdrsup.log.gz 2024-09-17T09:57:03.142 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-osd.3.log 2024-09-17T09:57:03.142 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph.log: 91.5% -- replaced with /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-17T09:57:03.142 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-17T09:57:03.144 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-osd.3.log: 87.1% -- replaced with /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-17T09:57:03.145 INFO:teuthology.orchestra.run.smithi136.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-osd.7.log 2024-09-17T09:57:03.145 INFO:teuthology.orchestra.run.smithi136.stderr:: No such file or directory 2024-09-17T09:57:03.145 INFO:teuthology.orchestra.run.smithi136.stderr:gzip -5 --verbose -- /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-mgr.smithi136.hicmlm.log 2024-09-17T09:57:03.145 INFO:teuthology.orchestra.run.smithi136.stderr:/var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-17T09:57:03.145 INFO:teuthology.orchestra.run.smithi136.stderr:/var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-mgr.smithi136.hicmlm.log: gzip -5 --verbose -- /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-osd.4.log 2024-09-17T09:57:03.145 INFO:teuthology.orchestra.run.smithi136.stderr:/var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-17T09:57:03.145 INFO:teuthology.orchestra.run.smithi136.stderr:gzip -5 --verbose -- /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi136.smslff.log 2024-09-17T09:57:03.145 INFO:teuthology.orchestra.run.smithi136.stderr:/var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-osd.4.log: /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-17T09:57:03.145 INFO:teuthology.orchestra.run.smithi136.stderr:/var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi136.smslff.log: gzip -5 --verbose -- /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-osd.6.log 2024-09-17T09:57:03.146 INFO:teuthology.orchestra.run.smithi136.stderr:/var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph.cephadm.log: 87.7% -- replaced with /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi136.smslff.log.gzgzip -5 --verbose -- /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-osd.5.log 2024-09-17T09:57:03.146 INFO:teuthology.orchestra.run.smithi136.stderr: 2024-09-17T09:57:03.146 INFO:teuthology.orchestra.run.smithi136.stderr:/var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-osd.6.log: 85.8% -- replaced with /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-17T09:57:03.146 INFO:teuthology.orchestra.run.smithi136.stderr:gzip -5 --verbose -- /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph.log 2024-09-17T09:57:03.146 INFO:teuthology.orchestra.run.smithi136.stderr:/var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-mon.smithi136.log 2024-09-17T09:57:03.146 INFO:teuthology.orchestra.run.smithi136.stderr:/var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph.log: 91.6% -- replaced with /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-17T09:57:03.146 INFO:teuthology.orchestra.run.smithi136.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-17T09:57:03.146 INFO:teuthology.orchestra.run.smithi136.stderr:/var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-mon.smithi136.log: 87.0% -- replaced with /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-17T09:57:03.149 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cephadm.log: 91.2% -- replaced with /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-17T09:57:03.150 INFO:teuthology.orchestra.run.smithi136.stderr:/var/log/ceph/cephadm.log: 91.2% -- replaced with /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-17T09:57:03.152 INFO:teuthology.orchestra.run.smithi136.stderr: 90.1% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-17T09:57:03.171 INFO:teuthology.orchestra.run.smithi078.stderr: 91.8% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-17T09:57:03.206 INFO:teuthology.orchestra.run.smithi136.stderr: 89.9% -- replaced with /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-mgr.smithi136.hicmlm.log.gz 2024-09-17T09:57:03.451 INFO:teuthology.orchestra.run.smithi136.stderr: 92.6% -- replaced with /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-mon.smithi136.log.gz 2024-09-17T09:57:03.515 INFO:teuthology.orchestra.run.smithi078.stderr: 88.7% -- replaced with /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-mgr.smithi078.rsqofi.log.gz 2024-09-17T09:57:05.015 INFO:teuthology.orchestra.run.smithi078.stderr: 91.9% -- replaced with /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-mon.smithi078.log.gz 2024-09-17T09:57:15.426 INFO:teuthology.orchestra.run.smithi078.stderr: 94.5% -- replaced with /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-osd.2.log.gz 2024-09-17T09:57:18.314 INFO:teuthology.orchestra.run.smithi136.stderr: 93.9% -- replaced with /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-osd.4.log.gz 2024-09-17T09:57:20.233 INFO:teuthology.orchestra.run.smithi078.stderr: 94.1% -- replaced with /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-osd.0.log.gz 2024-09-17T09:57:20.474 INFO:teuthology.orchestra.run.smithi136.stderr: 94.2% -- replaced with /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-osd.6.log.gz 2024-09-17T09:57:22.148 INFO:teuthology.orchestra.run.smithi078.stderr: 94.2% -- replaced with /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-osd.1.log.gz 2024-09-17T09:57:22.797 INFO:teuthology.orchestra.run.smithi136.stderr: 94.2% -- replaced with /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-osd.5.log.gz 2024-09-17T09:57:23.624 INFO:teuthology.orchestra.run.smithi078.stderr: 94.1% -- replaced with /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-osd.3.log.gz 2024-09-17T09:57:23.626 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-17T09:57:23.626 INFO:teuthology.orchestra.run.smithi078.stderr:real 0m20.503s 2024-09-17T09:57:23.626 INFO:teuthology.orchestra.run.smithi078.stderr:user 1m8.358s 2024-09-17T09:57:23.626 INFO:teuthology.orchestra.run.smithi078.stderr:sys 0m2.722s 2024-09-17T09:57:28.019 INFO:teuthology.orchestra.run.smithi136.stderr: 94.1% -- replaced with /var/log/ceph/35c0cd22-74d7-11ef-bceb-c7b262605968/ceph-osd.7.log.gz 2024-09-17T09:57:28.021 INFO:teuthology.orchestra.run.smithi136.stderr: 2024-09-17T09:57:28.021 INFO:teuthology.orchestra.run.smithi136.stderr:real 0m24.895s 2024-09-17T09:57:28.021 INFO:teuthology.orchestra.run.smithi136.stderr:user 1m14.320s 2024-09-17T09:57:28.021 INFO:teuthology.orchestra.run.smithi136.stderr:sys 0m2.963s 2024-09-17T09:57:28.021 INFO:tasks.cephadm:Archiving logs... 2024-09-17T09:57:28.022 DEBUG:teuthology.misc:Transferring archived files from smithi078:/var/log/ceph to /home/teuthworker/archive/avan-2024-09-17_09:09:13-orch:cephadm-wip-athakkar-testing-2024-09-17-0048-distro-default-smithi/7908750/remote/smithi078/log 2024-09-17T09:57:28.023 DEBUG:teuthology.orchestra.run.smithi078:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-17T09:57:37.690 DEBUG:teuthology.misc:Transferring archived files from smithi136:/var/log/ceph to /home/teuthworker/archive/avan-2024-09-17_09:09:13-orch:cephadm-wip-athakkar-testing-2024-09-17-0048-distro-default-smithi/7908750/remote/smithi136/log 2024-09-17T09:57:37.691 DEBUG:teuthology.orchestra.run.smithi136:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-17T09:57:48.326 INFO:tasks.cephadm:Removing cluster... 2024-09-17T09:57:48.326 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 --force 2024-09-17T09:57:48.690 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 35c0cd22-74d7-11ef-bceb-c7b262605968 --force 2024-09-17T09:57:49.074 INFO:tasks.cephadm:Removing cephadm ... 2024-09-17T09:57:49.074 DEBUG:teuthology.orchestra.run.smithi078:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-17T09:57:49.090 DEBUG:teuthology.orchestra.run.smithi136:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-17T09:57:49.105 INFO:tasks.cephadm:Teardown complete 2024-09-17T09:57:49.106 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-17T09:57:49.116 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-17T09:57:49.117 DEBUG:teuthology.orchestra.run.smithi078:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-17T09:57:49.132 DEBUG:teuthology.orchestra.run.smithi136:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-17T09:57:49.144 INFO:teuthology.orchestra.run.smithi078.stderr:bash: line 1: ntpq: command not found 2024-09-17T09:57:49.149 INFO:teuthology.orchestra.run.smithi078.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-17T09:57:49.149 INFO:teuthology.orchestra.run.smithi078.stdout:=============================================================================== 2024-09-17T09:57:49.149 INFO:teuthology.orchestra.run.smithi078.stdout:^+ hv01.front.sepia.ceph.com 3 6 377 6 -488us[ -449us] +/- 64ms 2024-09-17T09:57:49.149 INFO:teuthology.orchestra.run.smithi078.stdout:^* hv02.front.sepia.ceph.com 3 6 377 5 -447us[ -407us] +/- 53ms 2024-09-17T09:57:49.149 INFO:teuthology.orchestra.run.smithi078.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 4 +1251us[+1251us] +/- 73ms 2024-09-17T09:57:49.150 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T09:57:49.159 INFO:teuthology.orchestra.run.smithi136.stderr:bash: line 1: ntpq: command not found 2024-09-17T09:57:49.163 INFO:teuthology.orchestra.run.smithi136.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-17T09:57:49.163 INFO:teuthology.orchestra.run.smithi136.stdout:=============================================================================== 2024-09-17T09:57:49.164 INFO:teuthology.orchestra.run.smithi136.stdout:^+ hv01.front.sepia.ceph.com 3 6 377 2 -477us[ -477us] +/- 64ms 2024-09-17T09:57:49.164 INFO:teuthology.orchestra.run.smithi136.stdout:^* hv02.front.sepia.ceph.com 3 6 377 3 -426us[ -396us] +/- 53ms 2024-09-17T09:57:49.164 INFO:teuthology.orchestra.run.smithi136.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 1 +1244us[+1244us] +/- 73ms 2024-09-17T09:57:49.164 INFO:teuthology.orchestra.run.smithi136.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T09:57:49.164 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-17T09:57:49.174 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-17T09:57:49.175 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-17T09:57:49.184 DEBUG:teuthology.orchestra.run.smithi078:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-17T09:57:49.266 DEBUG:teuthology.orchestra.run.smithi136:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-17T09:57:49.337 DEBUG:teuthology.orchestra.run.smithi078:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-17T09:57:49.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T09:57:49.366 DEBUG:teuthology.orchestra.run.smithi136:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-17T09:57:49.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T09:57:49.394 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-17T09:57:49.404 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-17T09:57:49.486 INFO:teuthology.task.internal:Duration was 1985.847834 seconds 2024-09-17T09:57:49.487 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-17T09:57:49.522 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-17T09:57:49.522 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-17T09:57:49.525 DEBUG:teuthology.orchestra.run.smithi136:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-17T09:57:49.561 INFO:teuthology.orchestra.run.smithi078.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-17T09:57:49.563 INFO:teuthology.orchestra.run.smithi136.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-17T09:57:49.890 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-17T09:57:49.890 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi078.front.sepia.ceph.com 2024-09-17T09:57:49.891 DEBUG:teuthology.orchestra.run.smithi078:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-17T09:57:49.950 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi136.front.sepia.ceph.com 2024-09-17T09:57:49.951 DEBUG:teuthology.orchestra.run.smithi136:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-17T09:57:49.985 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-17T09:57:49.985 DEBUG:teuthology.orchestra.run.smithi078:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-17T09:57:49.993 DEBUG:teuthology.orchestra.run.smithi136:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-17T09:57:50.051 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-17T09:57:50.052 DEBUG:teuthology.orchestra.run.smithi078:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-17T09:57:50.060 DEBUG:teuthology.orchestra.run.smithi136:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-17T09:58:03.832 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-17T09:58:03.844 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-17T09:58:03.844 DEBUG:teuthology.orchestra.run.smithi078:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-17T09:58:03.870 DEBUG:teuthology.orchestra.run.smithi136:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-17T09:58:03.901 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-17T09:58:03.911 DEBUG:teuthology.orchestra.run.smithi078:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-17T09:58:03.913 DEBUG:teuthology.orchestra.run.smithi136:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-17T09:58:03.934 INFO:teuthology.orchestra.run.smithi078.stdout:kernel.core_pattern = core 2024-09-17T09:58:03.969 INFO:teuthology.orchestra.run.smithi136.stdout:kernel.core_pattern = core 2024-09-17T09:58:04.074 DEBUG:teuthology.orchestra.run.smithi078:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-17T09:58:04.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T09:58:04.089 DEBUG:teuthology.orchestra.run.smithi136:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-17T09:58:04.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T09:58:04.132 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-17T09:58:04.142 INFO:teuthology.task.internal:Transferring archived files... 2024-09-17T09:58:04.144 DEBUG:teuthology.misc:Transferring archived files from smithi078:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/avan-2024-09-17_09:09:13-orch:cephadm-wip-athakkar-testing-2024-09-17-0048-distro-default-smithi/7908750/remote/smithi078 2024-09-17T09:58:04.144 DEBUG:teuthology.orchestra.run.smithi078:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-17T09:58:04.207 DEBUG:teuthology.misc:Transferring archived files from smithi136:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/avan-2024-09-17_09:09:13-orch:cephadm-wip-athakkar-testing-2024-09-17-0048-distro-default-smithi/7908750/remote/smithi136 2024-09-17T09:58:04.207 DEBUG:teuthology.orchestra.run.smithi136:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-17T09:58:04.252 INFO:teuthology.task.internal:Removing archive directory... 2024-09-17T09:58:04.252 DEBUG:teuthology.orchestra.run.smithi078:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-17T09:58:04.255 DEBUG:teuthology.orchestra.run.smithi136:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-17T09:58:04.303 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-17T09:58:04.315 INFO:teuthology.task.internal:Not uploading archives. 2024-09-17T09:58:04.315 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-17T09:58:04.449 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-17T09:58:04.449 DEBUG:teuthology.orchestra.run.smithi078:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-17T09:58:04.451 DEBUG:teuthology.orchestra.run.smithi136:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-17T09:58:04.464 INFO:teuthology.orchestra.run.smithi078.stdout: 265325 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 17 09:58 /home/ubuntu/cephtest 2024-09-17T09:58:04.469 INFO:teuthology.orchestra.run.smithi136.stdout: 265324 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 17 09:58 /home/ubuntu/cephtest 2024-09-17T09:58:04.470 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-17T09:58:04.534 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-17T09:58:04.693 INFO:teuthology.run:Summary data: description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 1985.8478336334229 owner: scheduled_avan@teuthology success: true 2024-09-17T09:58:04.694 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-17T09:58:04.777 INFO:teuthology.run:pass