2024-09-16T09:18:27.951 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-16T09:18:27.968 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-16T09:18:28.056 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/avan-2024-09-16_08:23:53-orch:cephadm-wip-athakkar-testing-2024-09-12-1857-distro-default-smithi/7907040 branch: wip-athakkar-testing-2024-09-12-1857 description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: athakkar@redhat.com first_in_suite: false job_id: '7907040' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: avan-2024-09-16_08:23:53-orch:cephadm-wip-athakkar-testing-2024-09-12-1857-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-athakkar-testing-2024-09-12-1857 ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - CEPHADM_REFRESH_FAILED log-only-match: - CEPHADM_ sha1: 66e33d6af1496291b0602ee0a859e68145f17dad ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 66e33d6af1496291b0602ee0a859e68145f17dad selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: wip-athakkar-testing-2024-09-12-1857 sha1: 66e33d6af1496291b0602ee0a859e68145f17dad owner: scheduled_avan@teuthology priority: 80 repo: https://git.ceph.com/ceph-ci.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 4787 sha1: 66e33d6af1496291b0602ee0a859e68145f17dad sleep_before_teardown: 0 subset: 1/20 suite: orch:cephadm suite_branch: wip-athakkar-testing-2024-09-12-1857 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_66e33d6af1496291b0602ee0a859e68145f17dad/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 66e33d6af1496291b0602ee0a859e68145f17dad targets: smithi078.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCdEC6CZCWKUTNugb0S9VndJylQdRwXjLU/LjvLt3YzlLLnU0WRTdYKGdIcJkkAkhQpITl8DWdAqU120dIqnhyg= smithi088.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBM0d6g0ZAEsJ0hmmcBJyQtT6iWZsyHl9bzMu+JBTW1jj049sgh34mBQU8hMkLz/RqEkBVFYbl4pvdVylINNa77Q= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-16_08:23:53 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: avan verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-09-16T09:18:28.056 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_66e33d6af1496291b0602ee0a859e68145f17dad/qa; will attempt to use it 2024-09-16T09:18:28.057 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_66e33d6af1496291b0602ee0a859e68145f17dad/qa/tasks 2024-09-16T09:18:28.057 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-16T09:18:28.059 INFO:teuthology.task.internal:Checking packages... 2024-09-16T09:18:28.080 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '66e33d6af1496291b0602ee0a859e68145f17dad' 2024-09-16T09:18:28.081 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-16T09:18:28.081 INFO:teuthology.packaging:ref: None 2024-09-16T09:18:28.081 INFO:teuthology.packaging:tag: None 2024-09-16T09:18:28.081 INFO:teuthology.packaging:branch: wip-athakkar-testing-2024-09-12-1857 2024-09-16T09:18:28.081 INFO:teuthology.packaging:sha1: 66e33d6af1496291b0602ee0a859e68145f17dad 2024-09-16T09:18:28.081 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-athakkar-testing-2024-09-12-1857 2024-09-16T09:18:28.221 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-4882.g66e33d6a 2024-09-16T09:18:28.223 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-16T09:18:28.228 INFO:teuthology.task.internal:no buildpackages task found 2024-09-16T09:18:28.229 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-16T09:18:28.265 INFO:teuthology.task.internal:Saving configuration 2024-09-16T09:18:28.279 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-16T09:18:28.290 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-16T09:18:28.316 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi078.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/avan-2024-09-16_08:23:53-orch:cephadm-wip-athakkar-testing-2024-09-12-1857-distro-default-smithi/7907040', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-16 09:13:06.144264', 'locked_by': 'scheduled_avan@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCdEC6CZCWKUTNugb0S9VndJylQdRwXjLU/LjvLt3YzlLLnU0WRTdYKGdIcJkkAkhQpITl8DWdAqU120dIqnhyg='} 2024-09-16T09:18:28.341 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi088.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/avan-2024-09-16_08:23:53-orch:cephadm-wip-athakkar-testing-2024-09-12-1857-distro-default-smithi/7907040', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-16 09:13:06.142500', 'locked_by': 'scheduled_avan@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBM0d6g0ZAEsJ0hmmcBJyQtT6iWZsyHl9bzMu+JBTW1jj049sgh34mBQU8hMkLz/RqEkBVFYbl4pvdVylINNa77Q='} 2024-09-16T09:18:28.341 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-16T09:18:28.347 INFO:teuthology.task.internal:roles: ubuntu@smithi078.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-09-16T09:18:28.347 INFO:teuthology.task.internal:roles: ubuntu@smithi088.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-09-16T09:18:28.347 INFO:teuthology.run_tasks:Running task console_log... 2024-09-16T09:18:28.427 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fcfcd76fac0>, signals=[15]) 2024-09-16T09:18:28.428 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-16T09:18:28.433 INFO:teuthology.task.internal:Opening connections... 2024-09-16T09:18:28.433 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi078.front.sepia.ceph.com 2024-09-16T09:18:28.434 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-16T09:18:28.515 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi088.front.sepia.ceph.com 2024-09-16T09:18:28.516 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi088.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-16T09:18:28.589 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-16T09:18:28.595 DEBUG:teuthology.orchestra.run.smithi078:> uname -m 2024-09-16T09:18:28.614 INFO:teuthology.orchestra.run.smithi078.stdout:x86_64 2024-09-16T09:18:28.614 DEBUG:teuthology.orchestra.run.smithi078:> cat /etc/os-release 2024-09-16T09:18:28.672 INFO:teuthology.orchestra.run.smithi078.stdout:NAME="CentOS Stream" 2024-09-16T09:18:28.672 INFO:teuthology.orchestra.run.smithi078.stdout:VERSION="9" 2024-09-16T09:18:28.672 INFO:teuthology.orchestra.run.smithi078.stdout:ID="centos" 2024-09-16T09:18:28.672 INFO:teuthology.orchestra.run.smithi078.stdout:ID_LIKE="rhel fedora" 2024-09-16T09:18:28.672 INFO:teuthology.orchestra.run.smithi078.stdout:VERSION_ID="9" 2024-09-16T09:18:28.673 INFO:teuthology.orchestra.run.smithi078.stdout:PLATFORM_ID="platform:el9" 2024-09-16T09:18:28.673 INFO:teuthology.orchestra.run.smithi078.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-16T09:18:28.673 INFO:teuthology.orchestra.run.smithi078.stdout:ANSI_COLOR="0;31" 2024-09-16T09:18:28.673 INFO:teuthology.orchestra.run.smithi078.stdout:LOGO="fedora-logo-icon" 2024-09-16T09:18:28.673 INFO:teuthology.orchestra.run.smithi078.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-16T09:18:28.673 INFO:teuthology.orchestra.run.smithi078.stdout:HOME_URL="https://centos.org/" 2024-09-16T09:18:28.673 INFO:teuthology.orchestra.run.smithi078.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-16T09:18:28.673 INFO:teuthology.orchestra.run.smithi078.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-16T09:18:28.673 INFO:teuthology.orchestra.run.smithi078.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-16T09:18:28.674 INFO:teuthology.lock.ops:Updating smithi078.front.sepia.ceph.com on lock server 2024-09-16T09:18:28.697 DEBUG:teuthology.orchestra.run.smithi088:> uname -m 2024-09-16T09:18:28.716 INFO:teuthology.orchestra.run.smithi088.stdout:x86_64 2024-09-16T09:18:28.716 DEBUG:teuthology.orchestra.run.smithi088:> cat /etc/os-release 2024-09-16T09:18:28.774 INFO:teuthology.orchestra.run.smithi088.stdout:NAME="CentOS Stream" 2024-09-16T09:18:28.774 INFO:teuthology.orchestra.run.smithi088.stdout:VERSION="9" 2024-09-16T09:18:28.774 INFO:teuthology.orchestra.run.smithi088.stdout:ID="centos" 2024-09-16T09:18:28.774 INFO:teuthology.orchestra.run.smithi088.stdout:ID_LIKE="rhel fedora" 2024-09-16T09:18:28.774 INFO:teuthology.orchestra.run.smithi088.stdout:VERSION_ID="9" 2024-09-16T09:18:28.774 INFO:teuthology.orchestra.run.smithi088.stdout:PLATFORM_ID="platform:el9" 2024-09-16T09:18:28.774 INFO:teuthology.orchestra.run.smithi088.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-16T09:18:28.775 INFO:teuthology.orchestra.run.smithi088.stdout:ANSI_COLOR="0;31" 2024-09-16T09:18:28.775 INFO:teuthology.orchestra.run.smithi088.stdout:LOGO="fedora-logo-icon" 2024-09-16T09:18:28.775 INFO:teuthology.orchestra.run.smithi088.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-16T09:18:28.775 INFO:teuthology.orchestra.run.smithi088.stdout:HOME_URL="https://centos.org/" 2024-09-16T09:18:28.775 INFO:teuthology.orchestra.run.smithi088.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-16T09:18:28.775 INFO:teuthology.orchestra.run.smithi088.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-16T09:18:28.775 INFO:teuthology.orchestra.run.smithi088.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-16T09:18:28.775 INFO:teuthology.lock.ops:Updating smithi088.front.sepia.ceph.com on lock server 2024-09-16T09:18:28.797 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-16T09:18:28.805 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-16T09:18:28.831 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-16T09:18:28.832 DEBUG:teuthology.orchestra.run.smithi078:> test '!' -e /home/ubuntu/cephtest 2024-09-16T09:18:28.834 DEBUG:teuthology.orchestra.run.smithi088:> test '!' -e /home/ubuntu/cephtest 2024-09-16T09:18:28.850 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-16T09:18:28.856 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-16T09:18:28.856 DEBUG:teuthology.orchestra.run.smithi078:> test -z $(ls -A /var/lib/ceph) 2024-09-16T09:18:28.892 DEBUG:teuthology.orchestra.run.smithi088:> test -z $(ls -A /var/lib/ceph) 2024-09-16T09:18:28.914 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-16T09:18:28.969 INFO:teuthology.run_tasks:Running task kernel... 2024-09-16T09:18:28.983 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-16T09:18:28.983 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-16T09:18:28.984 DEBUG:teuthology.orchestra.run.smithi078:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-16T09:18:28.984 DEBUG:teuthology.orchestra.run.smithi088:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-16T09:18:29.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-16T09:18:29.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-16T09:18:29.001 DEBUG:teuthology.orchestra.run.smithi088:> uname -r 2024-09-16T09:18:29.002 DEBUG:teuthology.orchestra.run.smithi078:> uname -r 2024-09-16T09:18:29.058 INFO:teuthology.orchestra.run.smithi078.stdout:5.14.0-503.el9.x86_64 2024-09-16T09:18:29.059 INFO:teuthology.task.kernel:Running kernel on smithi078: 5.14.0-503.el9.x86_64 2024-09-16T09:18:29.059 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum install -y kernel 2024-09-16T09:18:29.060 INFO:teuthology.orchestra.run.smithi088.stdout:5.14.0-503.el9.x86_64 2024-09-16T09:18:29.060 INFO:teuthology.task.kernel:Running kernel on smithi088: 5.14.0-503.el9.x86_64 2024-09-16T09:18:29.060 DEBUG:teuthology.orchestra.run.smithi088:> sudo yum install -y kernel 2024-09-16T09:18:31.400 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - BaseOS 13 MB/s | 8.2 MB 00:00 2024-09-16T09:18:32.498 INFO:teuthology.orchestra.run.smithi088.stdout:CentOS Stream 9 - BaseOS 4.8 MB/s | 8.2 MB 00:01 2024-09-16T09:18:36.490 INFO:teuthology.orchestra.run.smithi088.stdout:CentOS Stream 9 - AppStream 7.1 MB/s | 20 MB 00:02 2024-09-16T09:18:40.433 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - AppStream 2.6 MB/s | 20 MB 00:07 2024-09-16T09:18:44.065 INFO:teuthology.orchestra.run.smithi088.stdout:CentOS Stream 9 - CRB 1.7 MB/s | 6.5 MB 00:03 2024-09-16T09:18:45.343 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - CRB 8.3 MB/s | 6.5 MB 00:00 2024-09-16T09:18:45.723 INFO:teuthology.orchestra.run.smithi088.stdout:CentOS Stream 9 - Extras packages 40 kB/s | 19 kB 00:00 2024-09-16T09:18:46.461 INFO:teuthology.orchestra.run.smithi088.stdout:Extra Packages for Enterprise Linux 37 MB/s | 23 MB 00:00 2024-09-16T09:18:46.827 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - Extras packages 70 kB/s | 19 kB 00:00 2024-09-16T09:18:47.906 INFO:teuthology.orchestra.run.smithi078.stdout:Extra Packages for Enterprise Linux 24 MB/s | 23 MB 00:00 2024-09-16T09:18:51.816 INFO:teuthology.orchestra.run.smithi088.stdout:lab-extras 34 kB/s | 1.7 kB 00:00 2024-09-16T09:18:53.428 INFO:teuthology.orchestra.run.smithi078.stdout:lab-extras 29 kB/s | 1.7 kB 00:00 2024-09-16T09:18:53.839 INFO:teuthology.orchestra.run.smithi088.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-16T09:18:53.839 INFO:teuthology.orchestra.run.smithi088.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-16T09:18:53.937 INFO:teuthology.orchestra.run.smithi088.stdout:Dependencies resolved. 2024-09-16T09:18:53.948 INFO:teuthology.orchestra.run.smithi088.stdout:================================================================================ 2024-09-16T09:18:53.948 INFO:teuthology.orchestra.run.smithi088.stdout: Package Architecture Version Repository Size 2024-09-16T09:18:53.948 INFO:teuthology.orchestra.run.smithi088.stdout:================================================================================ 2024-09-16T09:18:53.948 INFO:teuthology.orchestra.run.smithi088.stdout:Installing: 2024-09-16T09:18:53.949 INFO:teuthology.orchestra.run.smithi088.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-16T09:18:53.949 INFO:teuthology.orchestra.run.smithi088.stdout:Installing dependencies: 2024-09-16T09:18:53.949 INFO:teuthology.orchestra.run.smithi088.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-16T09:18:53.951 INFO:teuthology.orchestra.run.smithi088.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-16T09:18:53.952 INFO:teuthology.orchestra.run.smithi088.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-16T09:18:53.952 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:18:53.952 INFO:teuthology.orchestra.run.smithi088.stdout:Transaction Summary 2024-09-16T09:18:53.952 INFO:teuthology.orchestra.run.smithi088.stdout:================================================================================ 2024-09-16T09:18:53.952 INFO:teuthology.orchestra.run.smithi088.stdout:Install 4 Packages 2024-09-16T09:18:53.952 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:18:53.953 INFO:teuthology.orchestra.run.smithi088.stdout:Total download size: 79 M 2024-09-16T09:18:53.953 INFO:teuthology.orchestra.run.smithi088.stdout:Installed size: 126 M 2024-09-16T09:18:53.953 INFO:teuthology.orchestra.run.smithi088.stdout:Downloading Packages: 2024-09-16T09:18:54.176 INFO:teuthology.orchestra.run.smithi088.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 247 kB/s | 23 kB 00:00 2024-09-16T09:18:54.559 INFO:teuthology.orchestra.run.smithi088.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 33 MB/s | 16 MB 00:00 2024-09-16T09:18:54.943 INFO:teuthology.orchestra.run.smithi088.stdout:(3/4): kernel-modules-core-5.14.0-505.el9.x86_6 37 MB/s | 29 MB 00:00 2024-09-16T09:18:55.260 INFO:teuthology.orchestra.run.smithi088.stdout:(4/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 29 MB/s | 35 MB 00:01 2024-09-16T09:18:55.260 INFO:teuthology.orchestra.run.smithi088.stdout:-------------------------------------------------------------------------------- 2024-09-16T09:18:55.260 INFO:teuthology.orchestra.run.smithi088.stdout:Total 60 MB/s | 79 MB 00:01 2024-09-16T09:18:55.468 INFO:teuthology.orchestra.run.smithi078.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-16T09:18:55.468 INFO:teuthology.orchestra.run.smithi078.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-16T09:18:55.561 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-16T09:18:55.572 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-16T09:18:55.572 INFO:teuthology.orchestra.run.smithi078.stdout: Package Architecture Version Repository Size 2024-09-16T09:18:55.572 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-16T09:18:55.572 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-09-16T09:18:55.572 INFO:teuthology.orchestra.run.smithi078.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-16T09:18:55.572 INFO:teuthology.orchestra.run.smithi078.stdout:Installing dependencies: 2024-09-16T09:18:55.572 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-16T09:18:55.572 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-16T09:18:55.572 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-16T09:18:55.572 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-16T09:18:55.573 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-16T09:18:55.573 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-16T09:18:55.573 INFO:teuthology.orchestra.run.smithi078.stdout:Install 4 Packages 2024-09-16T09:18:55.573 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-16T09:18:55.573 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 79 M 2024-09-16T09:18:55.573 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 126 M 2024-09-16T09:18:55.574 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-16T09:18:55.680 INFO:teuthology.orchestra.run.smithi078.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 262 kB/s | 23 kB 00:00 2024-09-16T09:18:55.944 INFO:teuthology.orchestra.run.smithi088.stdout:Running transaction check 2024-09-16T09:18:56.231 INFO:teuthology.orchestra.run.smithi078.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 25 MB/s | 16 MB 00:00 2024-09-16T09:18:56.309 INFO:teuthology.orchestra.run.smithi088.stdout:Transaction check succeeded. 2024-09-16T09:18:56.310 INFO:teuthology.orchestra.run.smithi088.stdout:Running transaction test 2024-09-16T09:18:56.547 INFO:teuthology.orchestra.run.smithi078.stdout:(3/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 36 MB/s | 35 MB 00:00 2024-09-16T09:18:57.039 INFO:teuthology.orchestra.run.smithi078.stdout:(4/4): kernel-modules-core-5.14.0-505.el9.x86_6 21 MB/s | 29 MB 00:01 2024-09-16T09:18:57.039 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-16T09:18:57.039 INFO:teuthology.orchestra.run.smithi078.stdout:Total 54 MB/s | 79 MB 00:01 2024-09-16T09:18:57.272 INFO:teuthology.orchestra.run.smithi088.stdout:Transaction test succeeded. 2024-09-16T09:18:57.272 INFO:teuthology.orchestra.run.smithi088.stdout:Running transaction 2024-09-16T09:18:57.718 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-16T09:18:58.079 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-16T09:18:58.079 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-16T09:18:59.057 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-16T09:18:59.057 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-16T09:18:59.734 INFO:teuthology.orchestra.run.smithi088.stdout: Preparing : 1/1 2024-09-16T09:19:00.385 INFO:teuthology.orchestra.run.smithi088.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-16T09:19:00.470 INFO:teuthology.orchestra.run.smithi088.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-16T09:19:01.497 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-16T09:19:02.215 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-16T09:19:02.307 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-16T09:19:03.283 INFO:teuthology.orchestra.run.smithi088.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-16T09:19:03.448 INFO:teuthology.orchestra.run.smithi088.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-16T09:19:05.091 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-16T09:19:05.228 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-16T09:19:08.281 INFO:teuthology.orchestra.run.smithi088.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-16T09:19:08.323 INFO:teuthology.orchestra.run.smithi088.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-16T09:19:10.065 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-16T09:19:10.178 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-16T09:19:13.171 INFO:teuthology.orchestra.run.smithi088.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-16T09:19:15.009 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-16T09:19:51.312 INFO:teuthology.orchestra.run.smithi088.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-16T09:19:51.351 INFO:teuthology.orchestra.run.smithi088.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-16T09:19:51.717 INFO:teuthology.orchestra.run.smithi088.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-16T09:19:51.717 INFO:teuthology.orchestra.run.smithi088.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-16T09:19:51.718 INFO:teuthology.orchestra.run.smithi088.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-16T09:19:51.718 INFO:teuthology.orchestra.run.smithi088.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-16T09:19:52.526 INFO:teuthology.orchestra.run.smithi088.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-16T09:19:52.526 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:19:52.526 INFO:teuthology.orchestra.run.smithi088.stdout:Installed: 2024-09-16T09:19:52.526 INFO:teuthology.orchestra.run.smithi088.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-16T09:19:52.527 INFO:teuthology.orchestra.run.smithi088.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-16T09:19:52.527 INFO:teuthology.orchestra.run.smithi088.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-16T09:19:52.527 INFO:teuthology.orchestra.run.smithi088.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-16T09:19:52.527 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:19:52.527 INFO:teuthology.orchestra.run.smithi088.stdout:Complete! 2024-09-16T09:19:52.865 DEBUG:teuthology.orchestra.run.smithi088:> rpm -q kernel | sort -rV | head -n 1 2024-09-16T09:19:52.993 INFO:teuthology.orchestra.run.smithi088.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-16T09:19:52.994 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-16T09:19:52.994 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-16T09:19:52.994 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-16T09:19:52.994 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-16T09:19:52.994 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-09-16T09:19:52.995 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi088.front.sepia.ceph.com, path=None, version=distro) 2024-09-16T09:19:52.995 DEBUG:teuthology.orchestra.run.smithi088:> rpm -q kernel | sort -rV | head -n 1 2024-09-16T09:19:53.054 INFO:teuthology.orchestra.run.smithi088.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-16T09:19:53.054 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-16T09:19:53.054 DEBUG:teuthology.orchestra.run.smithi088:> sudo rpm -qi grub2-tools 2024-09-16T09:19:53.089 INFO:teuthology.orchestra.run.smithi088.stdout:Name : grub2-tools 2024-09-16T09:19:53.090 INFO:teuthology.orchestra.run.smithi088.stdout:Epoch : 1 2024-09-16T09:19:53.090 INFO:teuthology.orchestra.run.smithi088.stdout:Version : 2.06 2024-09-16T09:19:53.090 INFO:teuthology.orchestra.run.smithi088.stdout:Release : 82.el9 2024-09-16T09:19:53.090 INFO:teuthology.orchestra.run.smithi088.stdout:Architecture: x86_64 2024-09-16T09:19:53.090 INFO:teuthology.orchestra.run.smithi088.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-16T09:19:53.092 INFO:teuthology.orchestra.run.smithi088.stdout:Group : Unspecified 2024-09-16T09:19:53.092 INFO:teuthology.orchestra.run.smithi088.stdout:Size : 8274670 2024-09-16T09:19:53.092 INFO:teuthology.orchestra.run.smithi088.stdout:License : GPLv3+ 2024-09-16T09:19:53.092 INFO:teuthology.orchestra.run.smithi088.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-16T09:19:53.092 INFO:teuthology.orchestra.run.smithi088.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-16T09:19:53.092 INFO:teuthology.orchestra.run.smithi088.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-16T09:19:53.093 INFO:teuthology.orchestra.run.smithi088.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-16T09:19:53.093 INFO:teuthology.orchestra.run.smithi088.stdout:Packager : builder@centos.org 2024-09-16T09:19:53.093 INFO:teuthology.orchestra.run.smithi088.stdout:Vendor : CentOS 2024-09-16T09:19:53.093 INFO:teuthology.orchestra.run.smithi088.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-16T09:19:53.093 INFO:teuthology.orchestra.run.smithi088.stdout:Summary : Support tools for GRUB. 2024-09-16T09:19:53.093 INFO:teuthology.orchestra.run.smithi088.stdout:Description : 2024-09-16T09:19:53.093 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:19:53.093 INFO:teuthology.orchestra.run.smithi088.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-16T09:19:53.093 INFO:teuthology.orchestra.run.smithi088.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-16T09:19:53.093 INFO:teuthology.orchestra.run.smithi088.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-16T09:19:53.093 INFO:teuthology.orchestra.run.smithi088.stdout:hardware devices. 2024-09-16T09:19:53.093 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:19:53.094 INFO:teuthology.orchestra.run.smithi088.stdout:This subpackage provides tools for support of all platforms. 2024-09-16T09:19:53.095 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-16T09:19:53.095 INFO:teuthology.task.kernel:Updating grub on smithi088 to boot 5.14.0-505.el9.x86_64 2024-09-16T09:19:53.095 DEBUG:teuthology.orchestra.run.smithi088:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-16T09:19:53.264 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-16T09:19:53.304 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-16T09:19:53.659 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-16T09:19:53.660 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-16T09:19:53.660 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-16T09:19:53.660 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-16T09:19:53.734 INFO:teuthology.orchestra.run.smithi088.stderr:Generating grub configuration file ... 2024-09-16T09:19:54.481 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-16T09:19:54.481 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-16T09:19:54.481 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-16T09:19:54.481 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-16T09:19:54.481 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-16T09:19:54.481 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-16T09:19:54.481 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-16T09:19:54.481 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-16T09:19:54.481 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-16T09:19:54.846 DEBUG:teuthology.orchestra.run.smithi078:> rpm -q kernel | sort -rV | head -n 1 2024-09-16T09:19:54.923 INFO:teuthology.orchestra.run.smithi078.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-16T09:19:54.923 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-16T09:19:54.924 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-16T09:19:54.924 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-16T09:19:54.924 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-16T09:19:54.924 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-09-16T09:19:54.924 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi078.front.sepia.ceph.com, path=None, version=distro) 2024-09-16T09:19:54.924 DEBUG:teuthology.orchestra.run.smithi078:> rpm -q kernel | sort -rV | head -n 1 2024-09-16T09:19:54.982 INFO:teuthology.orchestra.run.smithi078.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-16T09:19:54.982 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-16T09:19:54.982 DEBUG:teuthology.orchestra.run.smithi078:> sudo rpm -qi grub2-tools 2024-09-16T09:19:55.020 INFO:teuthology.orchestra.run.smithi078.stdout:Name : grub2-tools 2024-09-16T09:19:55.020 INFO:teuthology.orchestra.run.smithi078.stdout:Epoch : 1 2024-09-16T09:19:55.020 INFO:teuthology.orchestra.run.smithi078.stdout:Version : 2.06 2024-09-16T09:19:55.021 INFO:teuthology.orchestra.run.smithi078.stdout:Release : 82.el9 2024-09-16T09:19:55.021 INFO:teuthology.orchestra.run.smithi078.stdout:Architecture: x86_64 2024-09-16T09:19:55.021 INFO:teuthology.orchestra.run.smithi078.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-16T09:19:55.021 INFO:teuthology.orchestra.run.smithi078.stdout:Group : Unspecified 2024-09-16T09:19:55.021 INFO:teuthology.orchestra.run.smithi078.stdout:Size : 8274670 2024-09-16T09:19:55.021 INFO:teuthology.orchestra.run.smithi078.stdout:License : GPLv3+ 2024-09-16T09:19:55.021 INFO:teuthology.orchestra.run.smithi078.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-16T09:19:55.021 INFO:teuthology.orchestra.run.smithi078.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-16T09:19:55.021 INFO:teuthology.orchestra.run.smithi078.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-16T09:19:55.021 INFO:teuthology.orchestra.run.smithi078.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-16T09:19:55.021 INFO:teuthology.orchestra.run.smithi078.stdout:Packager : builder@centos.org 2024-09-16T09:19:55.021 INFO:teuthology.orchestra.run.smithi078.stdout:Vendor : CentOS 2024-09-16T09:19:55.021 INFO:teuthology.orchestra.run.smithi078.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-16T09:19:55.022 INFO:teuthology.orchestra.run.smithi078.stdout:Summary : Support tools for GRUB. 2024-09-16T09:19:55.022 INFO:teuthology.orchestra.run.smithi078.stdout:Description : 2024-09-16T09:19:55.022 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-16T09:19:55.022 INFO:teuthology.orchestra.run.smithi078.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-16T09:19:55.022 INFO:teuthology.orchestra.run.smithi078.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-16T09:19:55.022 INFO:teuthology.orchestra.run.smithi078.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-16T09:19:55.022 INFO:teuthology.orchestra.run.smithi078.stdout:hardware devices. 2024-09-16T09:19:55.022 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-16T09:19:55.022 INFO:teuthology.orchestra.run.smithi078.stdout:This subpackage provides tools for support of all platforms. 2024-09-16T09:19:55.023 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-16T09:19:55.023 INFO:teuthology.task.kernel:Updating grub on smithi078 to boot 5.14.0-505.el9.x86_64 2024-09-16T09:19:55.023 DEBUG:teuthology.orchestra.run.smithi078:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-16T09:19:55.166 INFO:teuthology.orchestra.run.smithi088.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-16T09:19:55.197 INFO:teuthology.orchestra.run.smithi088.stderr:done 2024-09-16T09:19:55.200 DEBUG:teuthology.orchestra.run.smithi088:> mktemp 2024-09-16T09:19:55.215 INFO:teuthology.orchestra.run.smithi088.stdout:/tmp/tmp.UzrSOqwKCc 2024-09-16T09:19:55.216 DEBUG:teuthology.orchestra.run.smithi088:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.UzrSOqwKCc 2024-09-16T09:19:55.282 DEBUG:teuthology.orchestra.run.smithi088:> sudo chmod 0666 /tmp/tmp.UzrSOqwKCc 2024-09-16T09:19:55.416 DEBUG:teuthology.orchestra.remote:smithi088:/tmp/tmp.UzrSOqwKCc is 6KB 2024-09-16T09:19:55.464 DEBUG:teuthology.orchestra.run.smithi088:> rm -fr /tmp/tmp.UzrSOqwKCc 2024-09-16T09:19:55.479 DEBUG:teuthology.orchestra.run.smithi088:> sudo /bin/ls /boot/loader/entries || true 2024-09-16T09:19:55.542 INFO:teuthology.orchestra.run.smithi088.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-16T09:19:55.542 INFO:teuthology.orchestra.run.smithi088.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-16T09:19:55.542 INFO:teuthology.orchestra.run.smithi088.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-16T09:19:55.542 INFO:teuthology.orchestra.run.smithi088.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-16T09:19:55.544 DEBUG:teuthology.orchestra.run.smithi088:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-16T09:19:55.685 INFO:teuthology.orchestra.run.smithi078.stderr:Generating grub configuration file ... 2024-09-16T09:19:55.697 DEBUG:teuthology.orchestra.run.smithi088:> sudo shutdown -r now 2024-09-16T09:19:56.994 INFO:teuthology.orchestra.run.smithi078.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-16T09:19:57.019 INFO:teuthology.orchestra.run.smithi078.stderr:done 2024-09-16T09:19:57.021 DEBUG:teuthology.orchestra.run.smithi078:> mktemp 2024-09-16T09:19:57.036 INFO:teuthology.orchestra.run.smithi078.stdout:/tmp/tmp.Yf0jq7iTz6 2024-09-16T09:19:57.037 DEBUG:teuthology.orchestra.run.smithi078:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.Yf0jq7iTz6 2024-09-16T09:19:57.104 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 0666 /tmp/tmp.Yf0jq7iTz6 2024-09-16T09:19:57.264 DEBUG:teuthology.orchestra.remote:smithi078:/tmp/tmp.Yf0jq7iTz6 is 6KB 2024-09-16T09:19:57.313 DEBUG:teuthology.orchestra.run.smithi078:> rm -fr /tmp/tmp.Yf0jq7iTz6 2024-09-16T09:19:57.328 DEBUG:teuthology.orchestra.run.smithi078:> sudo /bin/ls /boot/loader/entries || true 2024-09-16T09:19:57.391 INFO:teuthology.orchestra.run.smithi078.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-16T09:19:57.391 INFO:teuthology.orchestra.run.smithi078.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-16T09:19:57.391 INFO:teuthology.orchestra.run.smithi078.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-16T09:19:57.391 INFO:teuthology.orchestra.run.smithi078.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-16T09:19:57.392 DEBUG:teuthology.orchestra.run.smithi078:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-16T09:19:57.544 DEBUG:teuthology.orchestra.run.smithi078:> sudo shutdown -r now 2024-09-16T09:20:25.728 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-09-16T09:20:25.728 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi088.front.sepia.ceph.com' 2024-09-16T09:20:25.729 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi088.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-16T09:20:27.549 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-09-16T09:20:27.549 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-16T09:20:27.550 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-16T09:20:44.097 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.88 2024-09-16T09:20:45.885 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.78 2024-09-16T09:20:53.105 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi088.front.sepia.ceph.com' 2024-09-16T09:20:53.106 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi088.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-16T09:20:54.888 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-16T09:20:54.889 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-16T09:20:56.157 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.88 2024-09-16T09:20:57.953 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.78 2024-09-16T09:21:08.241 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi088.front.sepia.ceph.com' 2024-09-16T09:21:08.241 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi088.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-16T09:21:09.956 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-16T09:21:09.956 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-16T09:21:11.293 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.88 2024-09-16T09:21:26.305 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi088.front.sepia.ceph.com' 2024-09-16T09:21:26.306 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi088.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-16T09:21:42.219 DEBUG:teuthology.orchestra.run.smithi078:> true 2024-09-16T09:21:42.769 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-16T09:21:42.769 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-505.el9.x86_64"... 2024-09-16T09:21:42.769 DEBUG:teuthology.orchestra.run.smithi078:> uname -r 2024-09-16T09:21:42.815 INFO:teuthology.orchestra.run.smithi078.stdout:5.14.0-505.el9.x86_64 2024-09-16T09:21:42.816 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-16T09:21:42.816 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-16T09:21:42.816 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-16T09:21:43.817 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-09-16T09:21:43.818 DEBUG:teuthology.orchestra.run.smithi078:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-16T09:21:43.877 INFO:teuthology.orchestra.run.smithi078.stdout:ttyS1 2024-09-16T09:21:43.936 DEBUG:teuthology.parallel:result is None 2024-09-16T09:21:58.144 DEBUG:teuthology.orchestra.run.smithi088:> true 2024-09-16T09:21:58.504 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi088.front.sepia.ceph.com' 2024-09-16T09:21:58.504 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-505.el9.x86_64"... 2024-09-16T09:21:58.505 DEBUG:teuthology.orchestra.run.smithi088:> uname -r 2024-09-16T09:21:58.521 INFO:teuthology.orchestra.run.smithi088.stdout:5.14.0-505.el9.x86_64 2024-09-16T09:21:58.521 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-16T09:21:58.521 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-16T09:21:58.521 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-16T09:21:59.523 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-09-16T09:21:59.523 DEBUG:teuthology.orchestra.run.smithi088:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-16T09:21:59.596 INFO:teuthology.orchestra.run.smithi088.stdout:ttyS1 2024-09-16T09:21:59.620 DEBUG:teuthology.parallel:result is None 2024-09-16T09:21:59.620 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-16T09:21:59.629 INFO:teuthology.task.internal:Creating test directory... 2024-09-16T09:21:59.629 DEBUG:teuthology.orchestra.run.smithi078:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-16T09:21:59.631 DEBUG:teuthology.orchestra.run.smithi088:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-16T09:21:59.650 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-16T09:21:59.683 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-16T09:21:59.717 INFO:teuthology.task.internal:Creating archive directory... 2024-09-16T09:21:59.717 DEBUG:teuthology.orchestra.run.smithi078:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-16T09:21:59.720 DEBUG:teuthology.orchestra.run.smithi088:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-16T09:21:59.766 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-16T09:21:59.777 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-16T09:21:59.777 DEBUG:teuthology.orchestra.run.smithi078:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-16T09:21:59.788 DEBUG:teuthology.orchestra.run.smithi088:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-16T09:21:59.827 INFO:teuthology.orchestra.run.smithi078.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-16T09:21:59.840 INFO:teuthology.orchestra.run.smithi078.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-16T09:21:59.840 INFO:teuthology.orchestra.run.smithi088.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-16T09:21:59.852 INFO:teuthology.orchestra.run.smithi088.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-16T09:21:59.854 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-16T09:21:59.861 INFO:teuthology.task.internal:Configuring sudo... 2024-09-16T09:21:59.861 DEBUG:teuthology.orchestra.run.smithi078:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-16T09:21:59.884 DEBUG:teuthology.orchestra.run.smithi088:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-16T09:21:59.923 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-16T09:21:59.934 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-16T09:21:59.934 DEBUG:teuthology.orchestra.run.smithi078:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-16T09:21:59.954 DEBUG:teuthology.orchestra.run.smithi088:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-16T09:21:59.981 DEBUG:teuthology.orchestra.run.smithi078:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-16T09:22:00.056 DEBUG:teuthology.orchestra.run.smithi078:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-16T09:22:00.110 DEBUG:teuthology.orchestra.run.smithi078:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-16T09:22:00.196 DEBUG:teuthology.orchestra.run.smithi078:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-16T09:22:00.242 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-16T09:22:00.242 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-16T09:22:00.312 DEBUG:teuthology.orchestra.run.smithi088:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-16T09:22:00.358 DEBUG:teuthology.orchestra.run.smithi088:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-16T09:22:00.414 DEBUG:teuthology.orchestra.run.smithi088:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-16T09:22:00.499 DEBUG:teuthology.orchestra.run.smithi088:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-16T09:22:00.543 DEBUG:teuthology.orchestra.run.smithi088:> set -ex 2024-09-16T09:22:00.543 DEBUG:teuthology.orchestra.run.smithi088:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-16T09:22:00.610 DEBUG:teuthology.orchestra.run.smithi078:> sudo service rsyslog restart 2024-09-16T09:22:00.612 DEBUG:teuthology.orchestra.run.smithi088:> sudo service rsyslog restart 2024-09-16T09:22:00.657 INFO:teuthology.orchestra.run.smithi078.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-16T09:22:00.708 INFO:teuthology.orchestra.run.smithi088.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-16T09:22:01.101 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-16T09:22:01.109 INFO:teuthology.task.internal:Starting timer... 2024-09-16T09:22:01.109 INFO:teuthology.run_tasks:Running task pcp... 2024-09-16T09:22:01.161 INFO:teuthology.run_tasks:Running task selinux... 2024-09-16T09:22:01.193 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-09-16T09:22:01.193 DEBUG:teuthology.orchestra.run.smithi078:> sudo service auditd rotate 2024-09-16T09:22:01.239 INFO:teuthology.orchestra.run.smithi078.stdout:Rotating logs: 2024-09-16T09:22:01.241 DEBUG:teuthology.orchestra.run.smithi088:> sudo service auditd rotate 2024-09-16T09:22:01.286 INFO:teuthology.orchestra.run.smithi088.stdout:Rotating logs: 2024-09-16T09:22:01.288 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-16T09:22:01.288 DEBUG:teuthology.orchestra.run.smithi078:> /usr/sbin/getenforce 2024-09-16T09:22:01.317 INFO:teuthology.orchestra.run.smithi078.stdout:Permissive 2024-09-16T09:22:01.318 DEBUG:teuthology.orchestra.run.smithi088:> /usr/sbin/getenforce 2024-09-16T09:22:01.347 INFO:teuthology.orchestra.run.smithi088.stdout:Permissive 2024-09-16T09:22:01.347 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi078.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi088.front.sepia.ceph.com': 'permissive'} 2024-09-16T09:22:01.348 DEBUG:teuthology.orchestra.run.smithi078:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-16T09:22:01.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-16T09:22:01.390 DEBUG:teuthology.orchestra.run.smithi088:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-16T09:22:01.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-16T09:22:01.420 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-16T09:22:01.421 DEBUG:teuthology.orchestra.run.smithi078:> sudo /usr/sbin/setenforce permissive 2024-09-16T09:22:01.466 DEBUG:teuthology.orchestra.run.smithi088:> sudo /usr/sbin/setenforce permissive 2024-09-16T09:22:01.502 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-16T09:22:01.514 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-16T09:22:01.526 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-16T09:22:01.527 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi078.front.sepia.ceph.com,smithi088.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-16T09:25:45.142 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi078.front.sepia.ceph.com'), Remote(name='ubuntu@smithi088.front.sepia.ceph.com')] 2024-09-16T09:25:45.144 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-16T09:25:45.145 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-16T09:25:45.225 DEBUG:teuthology.orchestra.run.smithi078:> true 2024-09-16T09:25:45.304 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-16T09:25:45.305 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi088.front.sepia.ceph.com' 2024-09-16T09:25:45.305 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi088.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-16T09:25:45.383 DEBUG:teuthology.orchestra.run.smithi088:> true 2024-09-16T09:25:45.465 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi088.front.sepia.ceph.com' 2024-09-16T09:25:45.465 INFO:teuthology.run_tasks:Running task clock... 2024-09-16T09:25:45.483 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-16T09:25:45.483 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-16T09:25:45.484 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-16T09:25:45.486 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-16T09:25:45.486 DEBUG:teuthology.orchestra.run.smithi088:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-16T09:25:45.512 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-16T09:25:45.526 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-16T09:25:45.535 INFO:teuthology.orchestra.run.smithi088.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-16T09:25:45.550 INFO:teuthology.orchestra.run.smithi088.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-16T09:25:45.550 INFO:teuthology.orchestra.run.smithi078.stderr:sudo: ntpd: command not found 2024-09-16T09:25:45.561 INFO:teuthology.orchestra.run.smithi078.stdout:506 Cannot talk to daemon 2024-09-16T09:25:45.574 INFO:teuthology.orchestra.run.smithi088.stderr:sudo: ntpd: command not found 2024-09-16T09:25:45.576 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-16T09:25:45.586 INFO:teuthology.orchestra.run.smithi088.stdout:506 Cannot talk to daemon 2024-09-16T09:25:45.590 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-16T09:25:45.600 INFO:teuthology.orchestra.run.smithi088.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-16T09:25:45.613 INFO:teuthology.orchestra.run.smithi088.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-16T09:25:45.642 INFO:teuthology.orchestra.run.smithi078.stderr:bash: line 1: ntpq: command not found 2024-09-16T09:25:45.645 INFO:teuthology.orchestra.run.smithi078.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-16T09:25:45.645 INFO:teuthology.orchestra.run.smithi078.stdout:=============================================================================== 2024-09-16T09:25:45.645 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-16T09:25:45.645 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-16T09:25:45.645 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-16T09:25:45.646 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-16T09:25:45.667 INFO:teuthology.orchestra.run.smithi088.stderr:bash: line 1: ntpq: command not found 2024-09-16T09:25:45.670 INFO:teuthology.orchestra.run.smithi088.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-16T09:25:45.670 INFO:teuthology.orchestra.run.smithi088.stdout:=============================================================================== 2024-09-16T09:25:45.671 INFO:teuthology.orchestra.run.smithi088.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-16T09:25:45.671 INFO:teuthology.orchestra.run.smithi088.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-16T09:25:45.671 INFO:teuthology.orchestra.run.smithi088.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-16T09:25:45.671 INFO:teuthology.orchestra.run.smithi088.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-16T09:25:45.672 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-16T09:25:45.777 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'CEPHADM_REFRESH_FAILED'], 'log-only-match': ['CEPHADM_'], 'sha1': '66e33d6af1496291b0602ee0a859e68145f17dad'} 2024-09-16T09:25:45.777 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-09-16T09:25:45.778 INFO:tasks.cephadm:Cluster fsid is a70c765e-740d-11ef-bceb-c7b262605968 2024-09-16T09:25:45.778 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-16T09:25:45.778 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-09-16T09:25:45.778 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi078': '172.21.15.78', 'mon.smithi088': '172.21.15.88'} 2024-09-16T09:25:45.778 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-16T09:25:45.779 DEBUG:teuthology.orchestra.run.smithi078:> sudo hostname $(hostname -s) 2024-09-16T09:25:45.805 DEBUG:teuthology.orchestra.run.smithi088:> sudo hostname $(hostname -s) 2024-09-16T09:25:45.830 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-09-16T09:25:45.830 DEBUG:teuthology.orchestra.run.smithi078:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-16T09:25:46.106 INFO:teuthology.orchestra.run.smithi078.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 16 09:25 /home/ubuntu/cephtest/cephadm 2024-09-16T09:25:46.107 DEBUG:teuthology.orchestra.run.smithi088:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-16T09:25:46.228 INFO:teuthology.orchestra.run.smithi088.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 16 09:25 /home/ubuntu/cephtest/cephadm 2024-09-16T09:25:46.228 DEBUG:teuthology.orchestra.run.smithi078:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-16T09:25:46.244 DEBUG:teuthology.orchestra.run.smithi088:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-16T09:25:46.316 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-09-16T09:25:46.316 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-16T09:25:46.319 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-16T09:25:47.868 INFO:teuthology.orchestra.run.smithi088.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-16T09:25:47.998 INFO:teuthology.orchestra.run.smithi078.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-16T09:25:59.717 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:25:59.718 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-16T09:25:59.718 INFO:teuthology.orchestra.run.smithi078.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-16T09:25:59.718 INFO:teuthology.orchestra.run.smithi078.stdout: "repo_digests": [ 2024-09-16T09:25:59.718 INFO:teuthology.orchestra.run.smithi078.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-16T09:25:59.718 INFO:teuthology.orchestra.run.smithi078.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-16T09:25:59.718 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-09-16T09:25:59.718 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:26:00.028 INFO:teuthology.orchestra.run.smithi088.stdout:{ 2024-09-16T09:26:00.028 INFO:teuthology.orchestra.run.smithi088.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-16T09:26:00.028 INFO:teuthology.orchestra.run.smithi088.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-16T09:26:00.028 INFO:teuthology.orchestra.run.smithi088.stdout: "repo_digests": [ 2024-09-16T09:26:00.029 INFO:teuthology.orchestra.run.smithi088.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-16T09:26:00.029 INFO:teuthology.orchestra.run.smithi088.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-16T09:26:00.029 INFO:teuthology.orchestra.run.smithi088.stdout: ] 2024-09-16T09:26:00.029 INFO:teuthology.orchestra.run.smithi088.stdout:} 2024-09-16T09:26:00.046 DEBUG:teuthology.orchestra.run.smithi078:> sudo mkdir -p /etc/ceph 2024-09-16T09:26:00.075 DEBUG:teuthology.orchestra.run.smithi088:> sudo mkdir -p /etc/ceph 2024-09-16T09:26:00.102 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 777 /etc/ceph 2024-09-16T09:26:00.141 DEBUG:teuthology.orchestra.run.smithi088:> sudo chmod 777 /etc/ceph 2024-09-16T09:26:00.166 INFO:tasks.cephadm:Writing seed config... 2024-09-16T09:26:00.167 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-16T09:26:00.168 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-16T09:26:00.168 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-16T09:26:00.168 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-16T09:26:00.168 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-16T09:26:00.168 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-16T09:26:00.168 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-16T09:26:00.168 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-16T09:26:00.169 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-16T09:26:00.169 DEBUG:teuthology.orchestra.run.smithi078:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-16T09:26:00.197 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = a70c765e-740d-11ef-bceb-c7b262605968 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-16T09:26:00.197 DEBUG:teuthology.orchestra.run.smithi078:mon.smithi078> sudo journalctl -f -n 0 -u ceph-a70c765e-740d-11ef-bceb-c7b262605968@mon.smithi078.service 2024-09-16T09:26:00.239 INFO:tasks.cephadm:Bootstrapping... 2024-09-16T09:26:00.239 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid a70c765e-740d-11ef-bceb-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.78 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-16T09:26:00.427 INFO:teuthology.orchestra.run.smithi078.stderr:-------------------------------------------------------------------------------- 2024-09-16T09:26:00.427 INFO:teuthology.orchestra.run.smithi078.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', 'a70c765e-740d-11ef-bceb-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.78', '--skip-admin-label'] 2024-09-16T09:26:00.448 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: 5.2.2 2024-09-16T09:26:00.450 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying podman|docker is present... 2024-09-16T09:26:00.470 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: 5.2.2 2024-09-16T09:26:00.472 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying lvm2 is present... 2024-09-16T09:26:00.472 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying time synchronization is in place... 2024-09-16T09:26:00.479 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-16T09:26:00.485 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2024-09-16T09:26:00.492 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: enabled 2024-09-16T09:26:00.498 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: active 2024-09-16T09:26:00.499 INFO:teuthology.orchestra.run.smithi078.stderr:Unit chronyd.service is enabled and running 2024-09-16T09:26:00.499 INFO:teuthology.orchestra.run.smithi078.stderr:Repeating the final host check... 2024-09-16T09:26:00.520 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: 5.2.2 2024-09-16T09:26:00.522 INFO:teuthology.orchestra.run.smithi078.stderr:podman (/bin/podman) version 5.2.2 is present 2024-09-16T09:26:00.522 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl is present 2024-09-16T09:26:00.522 INFO:teuthology.orchestra.run.smithi078.stderr:lvcreate is present 2024-09-16T09:26:00.528 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-16T09:26:00.534 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2024-09-16T09:26:00.541 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: enabled 2024-09-16T09:26:00.548 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: active 2024-09-16T09:26:00.548 INFO:teuthology.orchestra.run.smithi078.stderr:Unit chronyd.service is enabled and running 2024-09-16T09:26:00.549 INFO:teuthology.orchestra.run.smithi078.stderr:Host looks OK 2024-09-16T09:26:00.549 INFO:teuthology.orchestra.run.smithi078.stderr:Cluster fsid: a70c765e-740d-11ef-bceb-c7b262605968 2024-09-16T09:26:00.549 INFO:teuthology.orchestra.run.smithi078.stderr:Acquiring lock 140279063132048 on /run/cephadm/a70c765e-740d-11ef-bceb-c7b262605968.lock 2024-09-16T09:26:00.549 INFO:teuthology.orchestra.run.smithi078.stderr:Lock 140279063132048 acquired on /run/cephadm/a70c765e-740d-11ef-bceb-c7b262605968.lock 2024-09-16T09:26:00.549 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying IP 172.21.15.78 port 3300 ... 2024-09-16T09:26:00.550 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying IP 172.21.15.78 port 6789 ... 2024-09-16T09:26:00.550 INFO:teuthology.orchestra.run.smithi078.stderr:Base mon IP is 172.21.15.78, final addrv is [v2:172.21.15.78:3300,v1:172.21.15.78:6789] 2024-09-16T09:26:00.552 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.78 metric 100 2024-09-16T09:26:00.552 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.78 metric 100 2024-09-16T09:26:00.555 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-09-16T09:26:00.555 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2024-09-16T09:26:00.555 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2024-09-16T09:26:00.557 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-16T09:26:00.557 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-09-16T09:26:00.558 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-16T09:26:00.558 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-09-16T09:26:00.558 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe8f:d4dd/64 scope link noprefixroute 2024-09-16T09:26:00.558 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-16T09:26:00.558 INFO:teuthology.orchestra.run.smithi078.stderr:Mon IP `172.21.15.78` is in CIDR network `172.21.0.0/20` 2024-09-16T09:26:00.558 INFO:teuthology.orchestra.run.smithi078.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-16T09:26:00.559 INFO:teuthology.orchestra.run.smithi078.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-16T09:26:00.658 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-09-16T09:26:01.008 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Getting image source signatures 2024-09-16T09:26:01.008 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-09-16T09:26:01.008 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-09-16T09:26:01.008 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-09-16T09:26:01.008 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-09-16T09:26:01.008 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-09-16T09:26:01.008 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-16T09:26:01.179 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Writing manifest to image destination 2024-09-16T09:26:01.332 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-16T09:26:01.774 INFO:teuthology.orchestra.run.smithi078.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-16T09:26:02.330 INFO:teuthology.orchestra.run.smithi078.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-16T09:26:02.330 INFO:teuthology.orchestra.run.smithi078.stderr:Extracting ceph user uid/gid from container image... 2024-09-16T09:26:02.722 INFO:teuthology.orchestra.run.smithi078.stderr:stat: 167 167 2024-09-16T09:26:03.296 INFO:teuthology.orchestra.run.smithi078.stderr:Creating initial keys... 2024-09-16T09:26:03.654 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-authtool: AQCr+edmQDJ5JBAAT/SBMroSkc4M1LF9+ClRCg== 2024-09-16T09:26:04.518 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-authtool: AQCs+edmdefeGRAAXrG8PoKgBwGaeJt3yEhCEQ== 2024-09-16T09:26:05.465 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-authtool: AQCt+edmQDCgFhAAD6j7WyayXmMsGvlGWsEReA== 2024-09-16T09:26:06.014 INFO:teuthology.orchestra.run.smithi078.stderr:Creating initial monmap... 2024-09-16T09:26:06.446 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-16T09:26:06.446 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-09-16T09:26:06.446 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to a70c765e-740d-11ef-bceb-c7b262605968 2024-09-16T09:26:06.446 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-16T09:26:06.927 INFO:teuthology.orchestra.run.smithi078.stderr:monmaptool for smithi078 [v2:172.21.15.78:3300,v1:172.21.15.78:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-16T09:26:06.927 INFO:teuthology.orchestra.run.smithi078.stderr:setting min_mon_release = octopus 2024-09-16T09:26:06.928 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: set fsid to a70c765e-740d-11ef-bceb-c7b262605968 2024-09-16T09:26:06.928 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-16T09:26:06.928 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-16T09:26:06.928 INFO:teuthology.orchestra.run.smithi078.stderr:Creating mon... 2024-09-16T09:26:07.750 INFO:teuthology.orchestra.run.smithi078.stderr:create mon.smithi078 on 2024-09-16T09:26:07.961 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-16T09:26:08.129 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-a70c765e-740d-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-a70c765e-740d-11ef-bceb-c7b262605968.target. 2024-09-16T09:26:08.129 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-a70c765e-740d-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-a70c765e-740d-11ef-bceb-c7b262605968.target. 2024-09-16T09:26:08.435 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Failed to reset failed state of unit ceph-a70c765e-740d-11ef-bceb-c7b262605968@mon.smithi078.service: Unit ceph-a70c765e-740d-11ef-bceb-c7b262605968@mon.smithi078.service not loaded. 2024-09-16T09:26:08.442 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Created symlink /etc/systemd/system/ceph-a70c765e-740d-11ef-bceb-c7b262605968.target.wants/ceph-a70c765e-740d-11ef-bceb-c7b262605968@mon.smithi078.service → /etc/systemd/system/ceph-a70c765e-740d-11ef-bceb-c7b262605968@.service. 2024-09-16T09:26:09.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:08 smithi078 podman[23334]: 2024-09-16 09:26:08.730996793 +0000 UTC m=+0.016468392 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-16T09:26:10.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:09 smithi078 podman[23334]: 2024-09-16 09:26:09.669245293 +0000 UTC m=+0.954716883 container create b90e14358135dd44e982da535f4ef0561be1077616c4b7f5f03724b434079a7a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi078, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, name=centos-stream, io.openshift.expose-services=, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, release=754) 2024-09-16T09:26:11.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:11 smithi078 podman[23334]: 2024-09-16 09:26:11.484720747 +0000 UTC m=+2.770192346 container init b90e14358135dd44e982da535f4ef0561be1077616c4b7f5f03724b434079a7a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi078, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, release=754, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, distribution-scope=public, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, GIT_BRANCH=HEAD, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-16T09:26:11.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:11 smithi078 podman[23334]: 2024-09-16 09:26:11.488694247 +0000 UTC m=+2.774165839 container start b90e14358135dd44e982da535f4ef0561be1077616c4b7f5f03724b434079a7a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi078, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, RELEASE=HEAD, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, version=8, ceph=True, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., vcs-type=git) 2024-09-16T09:26:11.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:11 smithi078 ceph-mon[23356]: mkfs a70c765e-740d-11ef-bceb-c7b262605968 2024-09-16T09:26:11.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:11 smithi078 ceph-mon[23356]: mon.smithi078 is new leader, mons smithi078 in quorum (ranks 0) 2024-09-16T09:26:12.522 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: disabled 2024-09-16T09:26:12.530 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2024-09-16T09:26:12.530 INFO:teuthology.orchestra.run.smithi078.stderr:firewalld.service is not enabled 2024-09-16T09:26:12.530 INFO:teuthology.orchestra.run.smithi078.stderr:Not possible to enable service . firewalld.service is not available 2024-09-16T09:26:12.531 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mon to start... 2024-09-16T09:26:12.531 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mon... 2024-09-16T09:26:12.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:12 smithi078 bash[23334]: b90e14358135dd44e982da535f4ef0561be1077616c4b7f5f03724b434079a7a 2024-09-16T09:26:12.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:12 smithi078 systemd[1]: Started Ceph mon.smithi078 for a70c765e-740d-11ef-bceb-c7b262605968. 2024-09-16T09:26:12.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:12 smithi078 ceph-mon[23356]: mon.smithi078 is new leader, mons smithi078 in quorum (ranks 0) 2024-09-16T09:26:12.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:12 smithi078 ceph-mon[23356]: monmap e1: 1 mons at {smithi078=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0]} 2024-09-16T09:26:12.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:12 smithi078 ceph-mon[23356]: fsmap 2024-09-16T09:26:12.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:12 smithi078 ceph-mon[23356]: osdmap e1: 0 total, 0 up, 0 in 2024-09-16T09:26:12.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:12 smithi078 ceph-mon[23356]: mgrmap e1: no daemons active 2024-09-16T09:26:16.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:15 smithi078 ceph-mon[23356]: from='client.? 172.21.15.78:0/3811370522' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-16T09:26:16.617 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: cluster: 2024-09-16T09:26:16.617 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: id: a70c765e-740d-11ef-bceb-c7b262605968 2024-09-16T09:26:16.617 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-09-16T09:26:16.617 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-16T09:26:16.617 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: services: 2024-09-16T09:26:16.617 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi078 (age 4s) 2024-09-16T09:26:16.617 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mgr: no daemons active 2024-09-16T09:26:16.617 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-09-16T09:26:16.617 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-16T09:26:16.618 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: data: 2024-09-16T09:26:16.618 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-09-16T09:26:16.618 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-09-16T09:26:16.618 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-09-16T09:26:16.618 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: pgs: 2024-09-16T09:26:16.618 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-16T09:26:19.104 INFO:teuthology.orchestra.run.smithi078.stderr:mon is available 2024-09-16T09:26:19.104 INFO:teuthology.orchestra.run.smithi078.stderr:Assimilating anything we can from ceph.conf... 2024-09-16T09:26:19.612 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-16T09:26:19.612 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [global] 2024-09-16T09:26:19.613 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: fsid = a70c765e-740d-11ef-bceb-c7b262605968 2024-09-16T09:26:19.613 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.78:3300,v1:172.21.15.78:6789] 2024-09-16T09:26:19.613 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-16T09:26:19.613 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-16T09:26:19.613 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-16T09:26:19.613 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-16T09:26:19.613 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-16T09:26:19.613 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [mgr] 2024-09-16T09:26:19.613 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-16T09:26:19.613 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-16T09:26:19.613 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [osd] 2024-09-16T09:26:19.613 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-16T09:26:19.613 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-16T09:26:19.687 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:19 smithi078 ceph-mon[23356]: from='client.? 172.21.15.78:0/1309320666' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-16T09:26:19.687 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:19 smithi078 ceph-mon[23356]: from='client.? 172.21.15.78:0/1309320666' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-16T09:26:20.781 INFO:teuthology.orchestra.run.smithi078.stderr:Generating new minimal ceph.conf... 2024-09-16T09:26:21.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:21 smithi078 ceph-mon[23356]: from='client.? 172.21.15.78:0/2059017061' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:26:21.875 INFO:teuthology.orchestra.run.smithi078.stderr:Restarting the monitor... 2024-09-16T09:26:22.245 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:21 smithi078 systemd[1]: Stopping Ceph mon.smithi078 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:26:22.245 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:21 smithi078 bash[23574]: Error: no container with name or ID "ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon.smithi078" found: no such container 2024-09-16T09:26:22.245 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:22 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi078[23352]: 2024-09-16T09:26:22.049+0000 7f0d83b3a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi078 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T09:26:22.245 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:22 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi078[23352]: 2024-09-16T09:26:22.049+0000 7f0d83b3a700 -1 mon.smithi078@0(leader) e1 *** Got Signal Terminated *** 2024-09-16T09:26:22.245 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:22 smithi078 podman[23585]: 2024-09-16 09:26:22.244036697 +0000 UTC m=+0.315825809 container died b90e14358135dd44e982da535f4ef0561be1077616c4b7f5f03724b434079a7a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi078, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, RELEASE=HEAD, vcs-type=git, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, release=754, ceph=True, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, name=centos-stream, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-16T09:26:22.759 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:22 smithi078 podman[23585]: 2024-09-16 09:26:22.581839875 +0000 UTC m=+0.653628993 container cleanup b90e14358135dd44e982da535f4ef0561be1077616c4b7f5f03724b434079a7a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi078, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, name=centos-stream, vendor=Red Hat, Inc., distribution-scope=public, io.buildah.version=1.19.8, release=754, version=8, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, architecture=x86_64, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, ceph=True, RELEASE=HEAD, GIT_CLEAN=True) 2024-09-16T09:26:22.759 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:22 smithi078 bash[23585]: ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi078 2024-09-16T09:26:23.032 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:22 smithi078 podman[23597]: 2024-09-16 09:26:22.758043606 +0000 UTC m=+0.507644684 container remove b90e14358135dd44e982da535f4ef0561be1077616c4b7f5f03724b434079a7a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi078, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, maintainer=Guillaume Abrioux , ceph=True, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, name=centos-stream, distribution-scope=public, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD) 2024-09-16T09:26:23.033 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:22 smithi078 bash[23612]: Error: no container with name or ID "ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon.smithi078" found: no such container 2024-09-16T09:26:23.033 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:22 smithi078 systemd[1]: ceph-a70c765e-740d-11ef-bceb-c7b262605968@mon.smithi078.service: Deactivated successfully. 2024-09-16T09:26:23.033 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:22 smithi078 systemd[1]: Stopped Ceph mon.smithi078 for a70c765e-740d-11ef-bceb-c7b262605968. 2024-09-16T09:26:23.033 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:22 smithi078 systemd[1]: Starting Ceph mon.smithi078 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:26:23.033 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 podman[23676]: 2024-09-16 09:26:22.934730337 +0000 UTC m=+0.016564931 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-16T09:26:23.285 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 podman[23676]: 2024-09-16 09:26:23.059012239 +0000 UTC m=+0.140846832 container create c6b35fd1a57a7f7b09fbe05908c1d74b69ebc365f3970d351dc1b0f427235b81 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi078, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, GIT_BRANCH=HEAD, name=centos-stream, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, version=8, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , RELEASE=HEAD, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, io.openshift.expose-services=) 2024-09-16T09:26:23.285 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 podman[23676]: 2024-09-16 09:26:23.261777132 +0000 UTC m=+0.343611734 container init c6b35fd1a57a7f7b09fbe05908c1d74b69ebc365f3970d351dc1b0f427235b81 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi078, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, vcs-type=git, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-16T09:26:23.285 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 podman[23676]: 2024-09-16 09:26:23.265260168 +0000 UTC m=+0.347094762 container start c6b35fd1a57a7f7b09fbe05908c1d74b69ebc365f3970d351dc1b0f427235b81 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi078, ceph=True, io.buildah.version=1.19.8, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., distribution-scope=public, vcs-type=git, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, GIT_BRANCH=HEAD, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-16T09:26:23.365 INFO:teuthology.orchestra.run.smithi078.stderr:Setting mon public_network to 172.21.0.0/20 2024-09-16T09:26:23.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: set uid:gid to 167:167 (ceph:ceph) 2024-09-16T09:26:23.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-16T09:26:23.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: pidfile_write: ignore empty --pid-file 2024-09-16T09:26:23.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: load: jerasure load: lrc 2024-09-16T09:26:23.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: RocksDB version: 6.15.5 2024-09-16T09:26:23.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-16T09:26:23.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Compile date Apr 18 2022 2024-09-16T09:26:23.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: DB SUMMARY 2024-09-16T09:26:23.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: DB Session ID: 6HLYOPAR9V9MVH824PIF 2024-09-16T09:26:23.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: CURRENT file: CURRENT 2024-09-16T09:26:23.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: IDENTITY file: IDENTITY 2024-09-16T09:26:23.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-16T09:26:23.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi078/store.db dir, Total Num: 1, files: 000008.sst 2024-09-16T09:26:23.563 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi078/store.db: 000010.log size: 84010 ; 2024-09-16T09:26:23.563 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.error_if_exists: 0 2024-09-16T09:26:23.563 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.create_if_missing: 0 2024-09-16T09:26:23.563 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.paranoid_checks: 1 2024-09-16T09:26:23.563 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-16T09:26:23.563 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.env: 0x5646812c6860 2024-09-16T09:26:23.563 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.fs: Posix File System 2024-09-16T09:26:23.563 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.info_log: 0x564682c15fa0 2024-09-16T09:26:23.563 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.max_file_opening_threads: 16 2024-09-16T09:26:23.563 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.statistics: (nil) 2024-09-16T09:26:23.563 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.use_fsync: 0 2024-09-16T09:26:23.563 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.max_log_file_size: 0 2024-09-16T09:26:23.563 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-16T09:26:23.564 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-16T09:26:23.564 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.keep_log_file_num: 1000 2024-09-16T09:26:23.564 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.recycle_log_file_num: 0 2024-09-16T09:26:23.564 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.allow_fallocate: 1 2024-09-16T09:26:23.564 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.allow_mmap_reads: 0 2024-09-16T09:26:23.564 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.allow_mmap_writes: 0 2024-09-16T09:26:23.564 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.use_direct_reads: 0 2024-09-16T09:26:23.564 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-16T09:26:23.564 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.create_missing_column_families: 0 2024-09-16T09:26:23.564 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.db_log_dir: 2024-09-16T09:26:23.564 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi078/store.db 2024-09-16T09:26:23.564 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-16T09:26:23.565 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-16T09:26:23.566 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-16T09:26:23.566 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-16T09:26:23.566 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-16T09:26:23.566 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-16T09:26:23.566 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.advise_random_on_open: 1 2024-09-16T09:26:23.566 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.db_write_buffer_size: 0 2024-09-16T09:26:23.566 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.write_buffer_manager: 0x564682d06330 2024-09-16T09:26:23.566 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-16T09:26:23.567 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-16T09:26:23.567 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-16T09:26:23.567 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-16T09:26:23.567 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.rate_limiter: (nil) 2024-09-16T09:26:23.567 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-16T09:26:23.567 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.wal_recovery_mode: 2 2024-09-16T09:26:23.567 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.enable_thread_tracking: 0 2024-09-16T09:26:23.567 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.enable_pipelined_write: 0 2024-09-16T09:26:23.567 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.unordered_write: 0 2024-09-16T09:26:23.567 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-16T09:26:23.567 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-16T09:26:23.567 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-16T09:26:23.567 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-16T09:26:23.567 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.row_cache: None 2024-09-16T09:26:23.568 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.wal_filter: None 2024-09-16T09:26:23.568 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-16T09:26:23.568 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.allow_ingest_behind: 0 2024-09-16T09:26:23.568 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.preserve_deletes: 0 2024-09-16T09:26:23.568 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.two_write_queues: 0 2024-09-16T09:26:23.568 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.manual_wal_flush: 0 2024-09-16T09:26:23.568 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.atomic_flush: 0 2024-09-16T09:26:23.568 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-16T09:26:23.568 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-16T09:26:23.568 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-16T09:26:23.568 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.log_readahead_size: 0 2024-09-16T09:26:23.568 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-16T09:26:23.568 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.best_efforts_recovery: 0 2024-09-16T09:26:23.568 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-16T09:26:23.569 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-16T09:26:23.569 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.allow_data_in_errors: 0 2024-09-16T09:26:23.569 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.db_host_id: __hostname__ 2024-09-16T09:26:23.569 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.max_background_jobs: 2 2024-09-16T09:26:23.569 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.max_background_compactions: -1 2024-09-16T09:26:23.569 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.max_subcompactions: 1 2024-09-16T09:26:23.569 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-16T09:26:23.569 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-16T09:26:23.569 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-16T09:26:23.569 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.max_total_wal_size: 0 2024-09-16T09:26:23.569 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-16T09:26:23.569 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-16T09:26:23.569 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-16T09:26:23.570 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-16T09:26:23.570 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.max_open_files: -1 2024-09-16T09:26:23.570 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.bytes_per_sync: 0 2024-09-16T09:26:23.570 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-16T09:26:23.570 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-16T09:26:23.570 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.compaction_readahead_size: 0 2024-09-16T09:26:23.570 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.max_background_flushes: -1 2024-09-16T09:26:23.570 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Compression algorithms supported: 2024-09-16T09:26:23.570 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-16T09:26:23.570 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: kZSTD supported: 0 2024-09-16T09:26:23.570 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: kXpressCompression supported: 0 2024-09-16T09:26:23.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: kLZ4HCCompression supported: 1 2024-09-16T09:26:23.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: kLZ4Compression supported: 1 2024-09-16T09:26:23.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: kBZip2Compression supported: 0 2024-09-16T09:26:23.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: kZlibCompression supported: 1 2024-09-16T09:26:23.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: kSnappyCompression supported: 1 2024-09-16T09:26:23.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-16T09:26:23.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi078/store.db/MANIFEST-000009 2024-09-16T09:26:23.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-16T09:26:23.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-16T09:26:23.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.merge_operator: 2024-09-16T09:26:23.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.compaction_filter: None 2024-09-16T09:26:23.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.compaction_filter_factory: None 2024-09-16T09:26:23.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.sst_partitioner_factory: None 2024-09-16T09:26:23.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-16T09:26:23.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-16T09:26:23.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x564682be1d00) 2024-09-16T09:26:23.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: cache_index_and_filter_blocks: 1 2024-09-16T09:26:23.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-16T09:26:23.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-16T09:26:23.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: pin_top_level_index_and_filter: 1 2024-09-16T09:26:23.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: index_type: 0 2024-09-16T09:26:23.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: data_block_index_type: 0 2024-09-16T09:26:23.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: index_shortening: 1 2024-09-16T09:26:23.573 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-16T09:26:23.573 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: hash_index_allow_collision: 1 2024-09-16T09:26:23.573 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: checksum: 1 2024-09-16T09:26:23.573 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: no_block_cache: 0 2024-09-16T09:26:23.573 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache: 0x564682c4c170 2024-09-16T09:26:23.573 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache_name: BinnedLRUCache 2024-09-16T09:26:23.573 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache_options: 2024-09-16T09:26:23.573 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: capacity : 536870912 2024-09-16T09:26:23.573 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: num_shard_bits : 4 2024-09-16T09:26:23.573 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: strict_capacity_limit : 0 2024-09-16T09:26:23.573 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: high_pri_pool_ratio: 0.000 2024-09-16T09:26:23.573 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache_compressed: (nil) 2024-09-16T09:26:23.573 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: persistent_cache: (nil) 2024-09-16T09:26:23.574 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_size: 4096 2024-09-16T09:26:23.574 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_size_deviation: 10 2024-09-16T09:26:23.574 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_restart_interval: 16 2024-09-16T09:26:23.574 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: index_block_restart_interval: 1 2024-09-16T09:26:23.574 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: metadata_block_size: 4096 2024-09-16T09:26:23.574 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: partition_filters: 0 2024-09-16T09:26:23.574 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: use_delta_encoding: 1 2024-09-16T09:26:23.574 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-16T09:26:23.574 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: whole_key_filtering: 1 2024-09-16T09:26:23.575 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: verify_compression: 0 2024-09-16T09:26:23.575 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: read_amp_bytes_per_bit: 0 2024-09-16T09:26:23.575 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: format_version: 4 2024-09-16T09:26:23.575 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: enable_index_compression: 1 2024-09-16T09:26:23.575 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_align: 0 2024-09-16T09:26:23.575 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.write_buffer_size: 33554432 2024-09-16T09:26:23.575 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.max_write_buffer_number: 2 2024-09-16T09:26:23.575 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.compression: NoCompression 2024-09-16T09:26:23.575 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.bottommost_compression: Disabled 2024-09-16T09:26:23.575 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.prefix_extractor: nullptr 2024-09-16T09:26:23.576 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-16T09:26:23.576 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.num_levels: 7 2024-09-16T09:26:23.576 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-16T09:26:23.576 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-16T09:26:23.576 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-16T09:26:23.576 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-16T09:26:23.576 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-16T09:26:23.576 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-16T09:26:23.576 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-16T09:26:23.577 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-16T09:26:23.577 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-16T09:26:23.577 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-16T09:26:23.577 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-16T09:26:23.577 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.compression_opts.level: 32767 2024-09-16T09:26:23.578 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.compression_opts.strategy: 0 2024-09-16T09:26:23.578 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-16T09:26:23.578 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-16T09:26:23.578 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-16T09:26:23.578 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.compression_opts.enabled: false 2024-09-16T09:26:23.578 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-16T09:26:23.578 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-16T09:26:23.578 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-16T09:26:23.578 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.target_file_size_base: 67108864 2024-09-16T09:26:23.579 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-16T09:26:23.579 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-16T09:26:23.579 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-16T09:26:23.580 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-16T09:26:23.580 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-16T09:26:23.580 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-16T09:26:23.580 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-16T09:26:23.580 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-16T09:26:23.581 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-16T09:26:23.581 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-16T09:26:23.581 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-16T09:26:23.581 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-16T09:26:23.581 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-16T09:26:23.581 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.arena_block_size: 4194304 2024-09-16T09:26:23.582 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-16T09:26:23.582 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-16T09:26:23.582 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-16T09:26:23.582 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.disable_auto_compactions: 0 2024-09-16T09:26:23.582 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-16T09:26:23.582 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-16T09:26:23.582 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-16T09:26:23.582 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-16T09:26:23.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-16T09:26:23.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-16T09:26:23.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-16T09:26:23.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-16T09:26:23.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-16T09:26:23.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-16T09:26:23.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.table_properties_collectors: 2024-09-16T09:26:23.584 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.inplace_update_support: 0 2024-09-16T09:26:23.584 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-16T09:26:23.584 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-16T09:26:23.584 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-16T09:26:23.584 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-16T09:26:23.584 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.bloom_locality: 0 2024-09-16T09:26:23.584 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.max_successive_merges: 0 2024-09-16T09:26:23.584 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-16T09:26:23.585 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.paranoid_file_checks: 0 2024-09-16T09:26:23.585 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.force_consistency_checks: 1 2024-09-16T09:26:23.585 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.report_bg_io_stats: 0 2024-09-16T09:26:23.585 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.ttl: 2592000 2024-09-16T09:26:23.585 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-16T09:26:23.585 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.enable_blob_files: false 2024-09-16T09:26:23.586 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.min_blob_size: 0 2024-09-16T09:26:23.586 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.blob_file_size: 268435456 2024-09-16T09:26:23.586 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-16T09:26:23.586 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-16T09:26:23.586 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-16T09:26:23.586 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi078/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-16T09:26:23.586 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-09-16T09:26:23.587 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-09-16T09:26:23.587 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726478783291063, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-16T09:26:23.587 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-09-16T09:26:23.587 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-16T09:26:23.588 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726478783291752, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80095, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78340, "index_size": 243, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9946, "raw_average_key_size": 46, "raw_value_size": 72577, "raw_average_value_size": 340, "num_data_blocks": 11, "num_entries": 213, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726478783, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "1588589c-91b7-4c11-913a-ee25adeea195", "db_session_id": "6HLYOPAR9V9MVH824PIF"}} 2024-09-16T09:26:23.588 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-09-16T09:26:23.588 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726478783292129, "job": 1, "event": "recovery_finished"} 2024-09-16T09:26:23.588 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi078/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-16T09:26:23.588 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x564682c32a80 2024-09-16T09:26:23.588 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: DB pointer 0x564682c42000 2024-09-16T09:26:23.588 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-16T09:26:23.589 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-16T09:26:23.589 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** DB Stats ** 2024-09-16T09:26:23.589 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-16T09:26:23.589 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-16T09:26:23.589 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-16T09:26:23.589 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-16T09:26:23.589 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-16T09:26:23.589 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-16T09:26:23.589 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-16T09:26:23.589 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-16T09:26:23.589 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** Compaction Stats [default] ** 2024-09-16T09:26:23.589 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-16T09:26:23.590 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-16T09:26:23.590 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: L0 2/0 80.02 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 165.7 0.00 0.00 1 0.000 0 0 2024-09-16T09:26:23.590 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Sum 2/0 80.02 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 165.7 0.00 0.00 1 0.000 0 0 2024-09-16T09:26:23.590 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 165.7 0.00 0.00 1 0.000 0 0 2024-09-16T09:26:23.590 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-16T09:26:23.590 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** Compaction Stats [default] ** 2024-09-16T09:26:23.590 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-16T09:26:23.590 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-16T09:26:23.590 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 165.7 0.00 0.00 1 0.000 0 0 2024-09-16T09:26:23.590 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-16T09:26:23.590 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-16T09:26:23.590 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-16T09:26:23.590 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-16T09:26:23.590 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-16T09:26:23.591 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-16T09:26:23.591 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative compaction: 0.00 GB write, 24.93 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-16T09:26:23.591 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval compaction: 0.00 GB write, 24.93 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-16T09:26:23.591 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-16T09:26:23.591 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-16T09:26:23.591 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-16T09:26:23.591 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-16T09:26:23.591 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** Compaction Stats [default] ** 2024-09-16T09:26:23.591 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-16T09:26:23.591 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-16T09:26:23.592 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: L0 2/0 80.02 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 165.7 0.00 0.00 1 0.000 0 0 2024-09-16T09:26:23.592 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Sum 2/0 80.02 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 165.7 0.00 0.00 1 0.000 0 0 2024-09-16T09:26:23.592 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-16T09:26:23.592 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-16T09:26:23.592 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** Compaction Stats [default] ** 2024-09-16T09:26:23.592 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-16T09:26:23.592 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-16T09:26:23.592 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 165.7 0.00 0.00 1 0.000 0 0 2024-09-16T09:26:23.592 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-16T09:26:23.592 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-16T09:26:23.592 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-16T09:26:23.592 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-16T09:26:23.593 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-16T09:26:23.593 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-16T09:26:23.593 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative compaction: 0.00 GB write, 24.68 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-16T09:26:23.593 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-16T09:26:23.593 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-16T09:26:23.593 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-16T09:26:23.593 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-16T09:26:23.593 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: starting mon.smithi078 rank 0 at public addrs [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] at bind addrs [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi078 fsid a70c765e-740d-11ef-bceb-c7b262605968 2024-09-16T09:26:23.593 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: mon.smithi078@-1(???) e1 preinit fsid a70c765e-740d-11ef-bceb-c7b262605968 2024-09-16T09:26:23.593 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: mon.smithi078@-1(???).mds e1 new map 2024-09-16T09:26:23.593 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: mon.smithi078@-1(???).mds e1 print_map 2024-09-16T09:26:23.593 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: e1 2024-09-16T09:26:23.593 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-16T09:26:23.594 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-16T09:26:23.594 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: legacy client fscid: -1 2024-09-16T09:26:23.594 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-16T09:26:23.594 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: No filesystems configured 2024-09-16T09:26:23.594 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: mon.smithi078@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-16T09:26:23.594 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: mon.smithi078@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-16T09:26:23.594 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: mon.smithi078@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-16T09:26:23.594 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: mon.smithi078@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-16T09:26:23.594 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: mon.smithi078@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-16T09:26:23.594 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: expand_channel_meta expand map: {default=false} 2024-09-16T09:26:23.594 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: expand_channel_meta from 'false' to 'false' 2024-09-16T09:26:23.595 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: expand_channel_meta expanded map: {default=false} 2024-09-16T09:26:23.595 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: expand_channel_meta expand map: {default=info} 2024-09-16T09:26:23.595 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: expand_channel_meta from 'info' to 'info' 2024-09-16T09:26:23.595 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: expand_channel_meta expanded map: {default=info} 2024-09-16T09:26:23.595 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: expand_channel_meta expand map: {default=daemon} 2024-09-16T09:26:23.595 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: expand_channel_meta from 'daemon' to 'daemon' 2024-09-16T09:26:23.595 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: expand_channel_meta expanded map: {default=daemon} 2024-09-16T09:26:23.595 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: expand_channel_meta expand map: {default=debug} 2024-09-16T09:26:23.595 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: expand_channel_meta from 'debug' to 'debug' 2024-09-16T09:26:23.595 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: expand_channel_meta expanded map: {default=debug} 2024-09-16T09:26:23.595 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: mon.smithi078 is new leader, mons smithi078 in quorum (ranks 0) 2024-09-16T09:26:23.595 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: monmap e1: 1 mons at {smithi078=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0]} 2024-09-16T09:26:23.595 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: fsmap 2024-09-16T09:26:23.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: osdmap e1: 0 total, 0 up, 0 in 2024-09-16T09:26:23.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 ceph-mon[23694]: mgrmap e1: no daemons active 2024-09-16T09:26:23.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 bash[23676]: c6b35fd1a57a7f7b09fbe05908c1d74b69ebc365f3970d351dc1b0f427235b81 2024-09-16T09:26:23.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:23 smithi078 systemd[1]: Started Ceph mon.smithi078 for a70c765e-740d-11ef-bceb-c7b262605968. 2024-09-16T09:26:25.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:24 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/2246060575' entity='client.admin' 2024-09-16T09:26:26.256 INFO:teuthology.orchestra.run.smithi078.stderr:Wrote config to /etc/ceph/ceph.conf 2024-09-16T09:26:26.256 INFO:teuthology.orchestra.run.smithi078.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-16T09:26:26.256 INFO:teuthology.orchestra.run.smithi078.stderr:Creating mgr... 2024-09-16T09:26:26.256 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying port 9283 ... 2024-09-16T09:26:26.452 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Failed to reset failed state of unit ceph-a70c765e-740d-11ef-bceb-c7b262605968@mgr.smithi078.rrfxge.service: Unit ceph-a70c765e-740d-11ef-bceb-c7b262605968@mgr.smithi078.rrfxge.service not loaded. 2024-09-16T09:26:26.459 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Created symlink /etc/systemd/system/ceph-a70c765e-740d-11ef-bceb-c7b262605968.target.wants/ceph-a70c765e-740d-11ef-bceb-c7b262605968@mgr.smithi078.rrfxge.service → /etc/systemd/system/ceph-a70c765e-740d-11ef-bceb-c7b262605968@.service. 2024-09-16T09:26:27.151 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: disabled 2024-09-16T09:26:27.158 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2024-09-16T09:26:27.159 INFO:teuthology.orchestra.run.smithi078.stderr:firewalld.service is not enabled 2024-09-16T09:26:27.159 INFO:teuthology.orchestra.run.smithi078.stderr:Not possible to enable service . firewalld.service is not available 2024-09-16T09:26:27.167 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: disabled 2024-09-16T09:26:27.174 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2024-09-16T09:26:27.175 INFO:teuthology.orchestra.run.smithi078.stderr:firewalld.service is not enabled 2024-09-16T09:26:27.175 INFO:teuthology.orchestra.run.smithi078.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-09-16T09:26:27.175 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mgr to start... 2024-09-16T09:26:27.175 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mgr... 2024-09-16T09:26:27.722 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-16T09:26:27.723 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-09-16T09:26:27.723 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsid": "a70c765e-740d-11ef-bceb-c7b262605968", 2024-09-16T09:26:27.723 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "health": { 2024-09-16T09:26:27.723 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-16T09:26:27.723 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "checks": {}, 2024-09-16T09:26:27.723 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mutes": [] 2024-09-16T09:26:27.723 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-16T09:26:27.723 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-16T09:26:27.723 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum": [ 2024-09-16T09:26:27.723 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 0 2024-09-16T09:26:27.723 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-16T09:26:27.723 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-16T09:26:27.724 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "smithi078" 2024-09-16T09:26:27.724 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-16T09:26:27.724 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_age": 4, 2024-09-16T09:26:27.724 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "monmap": { 2024-09-16T09:26:27.724 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T09:26:27.724 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-16T09:26:27.724 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-16T09:26:27.724 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-16T09:26:27.724 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osdmap": { 2024-09-16T09:26:27.724 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T09:26:27.724 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-16T09:26:27.724 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-16T09:26:27.724 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-16T09:26:27.724 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-16T09:26:27.728 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-16T09:26:27.728 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-16T09:26:27.728 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-16T09:26:27.728 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgmap": { 2024-09-16T09:26:27.728 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-16T09:26:27.728 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-16T09:26:27.728 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-16T09:26:27.728 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-16T09:26:27.728 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-16T09:26:27.729 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-16T09:26:27.729 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-16T09:26:27.729 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-16T09:26:27.729 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-16T09:26:27.729 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsmap": { 2024-09-16T09:26:27.729 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T09:26:27.729 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-16T09:26:27.729 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-16T09:26:27.729 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-16T09:26:27.729 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-16T09:26:27.729 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": false, 2024-09-16T09:26:27.729 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-16T09:26:27.729 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modules": [ 2024-09-16T09:26:27.730 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "iostat", 2024-09-16T09:26:27.730 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "nfs", 2024-09-16T09:26:27.730 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "restful" 2024-09-16T09:26:27.730 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-16T09:26:27.730 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-09-16T09:26:27.730 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-16T09:26:27.730 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "servicemap": { 2024-09-16T09:26:27.730 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T09:26:27.730 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modified": "2024-09-16T09:26:11.530421+0000", 2024-09-16T09:26:27.730 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-09-16T09:26:27.730 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-16T09:26:27.730 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-16T09:26:27.730 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-09-16T09:26:28.011 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:27 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/2394254492' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-16T09:26:28.322 INFO:teuthology.orchestra.run.smithi078.stderr:mgr not available, waiting (1/15)... 2024-09-16T09:26:30.811 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-16T09:26:30.811 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-09-16T09:26:30.811 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsid": "a70c765e-740d-11ef-bceb-c7b262605968", 2024-09-16T09:26:30.812 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "health": { 2024-09-16T09:26:30.812 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-16T09:26:30.812 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "checks": {}, 2024-09-16T09:26:30.812 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mutes": [] 2024-09-16T09:26:30.812 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-16T09:26:30.812 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-16T09:26:30.812 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum": [ 2024-09-16T09:26:30.812 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 0 2024-09-16T09:26:30.812 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-16T09:26:30.812 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-16T09:26:30.812 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "smithi078" 2024-09-16T09:26:30.812 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-16T09:26:30.812 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_age": 7, 2024-09-16T09:26:30.813 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "monmap": { 2024-09-16T09:26:30.813 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T09:26:30.813 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-16T09:26:30.813 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-16T09:26:30.813 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-16T09:26:30.813 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osdmap": { 2024-09-16T09:26:30.813 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T09:26:30.813 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-16T09:26:30.813 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-16T09:26:30.813 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-16T09:26:30.814 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-16T09:26:30.814 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-16T09:26:30.814 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-16T09:26:30.814 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-16T09:26:30.814 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgmap": { 2024-09-16T09:26:30.814 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-16T09:26:30.814 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-16T09:26:30.814 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-16T09:26:30.814 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-16T09:26:30.814 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-16T09:26:30.814 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-16T09:26:30.815 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-16T09:26:30.815 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-16T09:26:30.815 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-16T09:26:30.815 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsmap": { 2024-09-16T09:26:30.815 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T09:26:30.815 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-16T09:26:30.816 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-16T09:26:30.816 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-16T09:26:30.816 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-16T09:26:30.816 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": false, 2024-09-16T09:26:30.816 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-16T09:26:30.816 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modules": [ 2024-09-16T09:26:30.817 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "iostat", 2024-09-16T09:26:30.817 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "nfs", 2024-09-16T09:26:30.817 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "restful" 2024-09-16T09:26:30.817 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-16T09:26:30.817 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-09-16T09:26:30.817 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-16T09:26:30.818 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "servicemap": { 2024-09-16T09:26:30.818 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T09:26:30.818 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modified": "2024-09-16T09:26:11.530421+0000", 2024-09-16T09:26:30.818 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-09-16T09:26:30.818 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-16T09:26:30.819 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-16T09:26:30.819 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-09-16T09:26:30.986 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:30 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/2860307626' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-16T09:26:32.304 INFO:teuthology.orchestra.run.smithi078.stderr:mgr not available, waiting (2/15)... 2024-09-16T09:26:33.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:33 smithi078 ceph-mon[23694]: Activating manager daemon smithi078.rrfxge 2024-09-16T09:26:33.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:33 smithi078 ceph-mon[23694]: mgrmap e2: smithi078.rrfxge(active, starting, since 0.0029154s) 2024-09-16T09:26:33.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:33 smithi078 ceph-mon[23694]: from='mgr.14100 172.21.15.78:0/2156699985' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-16T09:26:33.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:33 smithi078 ceph-mon[23694]: from='mgr.14100 172.21.15.78:0/2156699985' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-16T09:26:33.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:33 smithi078 ceph-mon[23694]: from='mgr.14100 172.21.15.78:0/2156699985' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-16T09:26:33.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:33 smithi078 ceph-mon[23694]: from='mgr.14100 172.21.15.78:0/2156699985' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-16T09:26:33.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:33 smithi078 ceph-mon[23694]: from='mgr.14100 172.21.15.78:0/2156699985' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mgr metadata", "who": "smithi078.rrfxge", "id": "smithi078.rrfxge"}]: dispatch 2024-09-16T09:26:33.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:33 smithi078 ceph-mon[23694]: Manager daemon smithi078.rrfxge is now available 2024-09-16T09:26:33.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:33 smithi078 ceph-mon[23694]: from='mgr.14100 172.21.15.78:0/2156699985' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:26:33.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:33 smithi078 ceph-mon[23694]: from='mgr.14100 172.21.15.78:0/2156699985' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/trash_purge_schedule"}]: dispatch 2024-09-16T09:26:33.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:33 smithi078 ceph-mon[23694]: from='mgr.14100 172.21.15.78:0/2156699985' entity='mgr.smithi078.rrfxge' 2024-09-16T09:26:33.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:33 smithi078 ceph-mon[23694]: from='mgr.14100 172.21.15.78:0/2156699985' entity='mgr.smithi078.rrfxge' 2024-09-16T09:26:33.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:33 smithi078 ceph-mon[23694]: from='mgr.14100 172.21.15.78:0/2156699985' entity='mgr.smithi078.rrfxge' 2024-09-16T09:26:34.949 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-16T09:26:34.949 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-09-16T09:26:34.949 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsid": "a70c765e-740d-11ef-bceb-c7b262605968", 2024-09-16T09:26:34.949 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "health": { 2024-09-16T09:26:34.950 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-16T09:26:34.950 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "checks": {}, 2024-09-16T09:26:34.950 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mutes": [] 2024-09-16T09:26:34.950 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-16T09:26:34.950 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-16T09:26:34.950 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum": [ 2024-09-16T09:26:34.950 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 0 2024-09-16T09:26:34.950 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-16T09:26:34.951 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-16T09:26:34.951 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "smithi078" 2024-09-16T09:26:34.951 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-16T09:26:34.951 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_age": 11, 2024-09-16T09:26:34.951 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "monmap": { 2024-09-16T09:26:34.951 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T09:26:34.951 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-16T09:26:34.951 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-16T09:26:34.952 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-16T09:26:34.952 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osdmap": { 2024-09-16T09:26:34.952 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T09:26:34.952 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-16T09:26:34.952 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-16T09:26:34.952 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-16T09:26:34.952 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-16T09:26:34.956 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-16T09:26:34.956 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-16T09:26:34.956 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-16T09:26:34.956 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgmap": { 2024-09-16T09:26:34.956 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-16T09:26:34.956 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-16T09:26:34.956 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-16T09:26:34.956 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-16T09:26:34.956 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-16T09:26:34.957 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-16T09:26:34.957 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-16T09:26:34.957 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-16T09:26:34.957 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-16T09:26:34.957 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsmap": { 2024-09-16T09:26:34.957 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T09:26:34.957 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-16T09:26:34.957 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-16T09:26:34.957 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-16T09:26:34.958 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-16T09:26:34.958 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": true, 2024-09-16T09:26:34.958 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-16T09:26:34.958 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modules": [ 2024-09-16T09:26:34.958 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "iostat", 2024-09-16T09:26:34.958 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "nfs", 2024-09-16T09:26:34.959 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "restful" 2024-09-16T09:26:34.959 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-16T09:26:34.959 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-09-16T09:26:34.959 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-16T09:26:34.959 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "servicemap": { 2024-09-16T09:26:34.959 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T09:26:34.960 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modified": "2024-09-16T09:26:11.530421+0000", 2024-09-16T09:26:34.960 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-09-16T09:26:34.960 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-16T09:26:34.960 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-16T09:26:34.960 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-09-16T09:26:35.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:35 smithi078 ceph-mon[23694]: mgrmap e3: smithi078.rrfxge(active, since 1.00643s) 2024-09-16T09:26:35.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:35 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/130140294' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-16T09:26:35.543 INFO:teuthology.orchestra.run.smithi078.stderr:mgr is available 2024-09-16T09:26:36.134 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-16T09:26:36.134 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [global] 2024-09-16T09:26:36.134 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: fsid = a70c765e-740d-11ef-bceb-c7b262605968 2024-09-16T09:26:36.134 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-16T09:26:36.134 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-16T09:26:36.134 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-16T09:26:36.135 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-16T09:26:36.135 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-16T09:26:36.135 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [mgr] 2024-09-16T09:26:36.135 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-16T09:26:36.135 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-16T09:26:36.135 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [osd] 2024-09-16T09:26:36.135 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-16T09:26:36.135 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-16T09:26:36.430 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:36 smithi078 ceph-mon[23694]: mgrmap e4: smithi078.rrfxge(active, since 2s) 2024-09-16T09:26:36.430 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:36 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/3645913292' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-16T09:26:36.633 INFO:teuthology.orchestra.run.smithi078.stderr:Enabling cephadm module... 2024-09-16T09:26:37.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:37 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/2124875576' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-16T09:26:38.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:38 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/2124875576' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-16T09:26:38.486 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:38 smithi078 ceph-mon[23694]: mgrmap e5: smithi078.rrfxge(active, since 4s) 2024-09-16T09:26:38.605 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-09-16T09:26:38.605 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 5, 2024-09-16T09:26:38.605 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": true, 2024-09-16T09:26:38.605 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "active_name": "smithi078.rrfxge", 2024-09-16T09:26:38.605 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-16T09:26:38.605 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-09-16T09:26:39.128 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for the mgr to restart... 2024-09-16T09:26:39.128 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mgr epoch 5... 2024-09-16T09:26:39.421 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:39 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/3621024586' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-16T09:26:44.779 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:44 smithi078 ceph-mon[23694]: Active manager daemon smithi078.rrfxge restarted 2024-09-16T09:26:44.779 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:44 smithi078 ceph-mon[23694]: Activating manager daemon smithi078.rrfxge 2024-09-16T09:26:44.779 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:44 smithi078 ceph-mon[23694]: osdmap e2: 0 total, 0 up, 0 in 2024-09-16T09:26:44.779 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:44 smithi078 ceph-mon[23694]: mgrmap e6: smithi078.rrfxge(active, starting, since 0.958667s) 2024-09-16T09:26:44.779 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:44 smithi078 ceph-mon[23694]: from='mgr.14118 172.21.15.78:0/3051481559' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-16T09:26:44.779 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:44 smithi078 ceph-mon[23694]: from='mgr.14118 172.21.15.78:0/3051481559' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mgr metadata", "who": "smithi078.rrfxge", "id": "smithi078.rrfxge"}]: dispatch 2024-09-16T09:26:44.779 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:44 smithi078 ceph-mon[23694]: from='mgr.14118 172.21.15.78:0/3051481559' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-16T09:26:44.779 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:44 smithi078 ceph-mon[23694]: from='mgr.14118 172.21.15.78:0/3051481559' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-16T09:26:44.780 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:44 smithi078 ceph-mon[23694]: from='mgr.14118 172.21.15.78:0/3051481559' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-16T09:26:44.780 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:44 smithi078 ceph-mon[23694]: Manager daemon smithi078.rrfxge is now available 2024-09-16T09:26:44.780 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:44 smithi078 ceph-mon[23694]: from='mgr.14118 172.21.15.78:0/3051481559' entity='mgr.smithi078.rrfxge' 2024-09-16T09:26:44.780 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:44 smithi078 ceph-mon[23694]: from='mgr.14118 172.21.15.78:0/3051481559' entity='mgr.smithi078.rrfxge' 2024-09-16T09:26:44.780 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:44 smithi078 ceph-mon[23694]: from='mgr.14118 172.21.15.78:0/3051481559' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:26:44.780 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:44 smithi078 ceph-mon[23694]: from='mgr.14118 172.21.15.78:0/3051481559' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:26:44.780 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:44 smithi078 ceph-mon[23694]: from='mgr.14118 172.21.15.78:0/3051481559' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:26:45.312 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-09-16T09:26:45.313 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-09-16T09:26:45.313 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "initialized": true 2024-09-16T09:26:45.313 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-09-16T09:26:45.531 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:45 smithi078 ceph-mon[23694]: from='mgr.14118 172.21.15.78:0/3051481559' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:26:45.531 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:45 smithi078 ceph-mon[23694]: from='mgr.14118 172.21.15.78:0/3051481559' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/trash_purge_schedule"}]: dispatch 2024-09-16T09:26:45.531 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:45 smithi078 ceph-mon[23694]: from='mgr.14118 172.21.15.78:0/3051481559' entity='mgr.smithi078.rrfxge' 2024-09-16T09:26:45.531 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:45 smithi078 ceph-mon[23694]: mgrmap e7: smithi078.rrfxge(active, since 1.96083s) 2024-09-16T09:26:45.531 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:45 smithi078 ceph-mon[23694]: from='mgr.14118 172.21.15.78:0/3051481559' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:26:45.884 INFO:teuthology.orchestra.run.smithi078.stderr:mgr epoch 5 is available 2024-09-16T09:26:45.884 INFO:teuthology.orchestra.run.smithi078.stderr:Setting orchestrator backend to cephadm... 2024-09-16T09:26:46.521 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:46 smithi078 ceph-mon[23694]: [16/Sep/2024:09:26:45] ENGINE Bus STARTING 2024-09-16T09:26:46.522 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:46 smithi078 ceph-mon[23694]: [16/Sep/2024:09:26:45] ENGINE Serving on https://172.21.15.78:7150 2024-09-16T09:26:46.522 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:46 smithi078 ceph-mon[23694]: [16/Sep/2024:09:26:45] ENGINE Bus STARTED 2024-09-16T09:26:46.522 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:46 smithi078 ceph-mon[23694]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-16T09:26:46.522 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:46 smithi078 ceph-mon[23694]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-16T09:26:46.522 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:46 smithi078 ceph-mon[23694]: mgrmap e8: smithi078.rrfxge(active, since 2s) 2024-09-16T09:26:47.669 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: value unchanged 2024-09-16T09:26:47.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:47 smithi078 ceph-mon[23694]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:26:47.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:47 smithi078 ceph-mon[23694]: from='mgr.14118 172.21.15.78:0/3051481559' entity='mgr.smithi078.rrfxge' 2024-09-16T09:26:47.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:47 smithi078 ceph-mon[23694]: from='mgr.14118 172.21.15.78:0/3051481559' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:26:48.196 INFO:teuthology.orchestra.run.smithi078.stderr:Generating ssh key... 2024-09-16T09:26:49.466 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:49 smithi078 ceph-mon[23694]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:26:49.466 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:49 smithi078 ceph-mon[23694]: from='mgr.14118 172.21.15.78:0/3051481559' entity='mgr.smithi078.rrfxge' 2024-09-16T09:26:49.466 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:49 smithi078 ceph-mon[23694]: from='mgr.14118 172.21.15.78:0/3051481559' entity='mgr.smithi078.rrfxge' 2024-09-16T09:26:50.057 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDnRbVwS/ERhvRSm6fncziAziD85QZvoTIWYudopUdrbEOyGXYEavCndGoTX+FTrrvgeg9Qszr0HvrtgHi5o9hpzPlLHLI4R8Q0hTss1yC1lC95BnQGWVb8Ya7z03dLJfQGsSkeFPdMv2U8xkGTdEp4btKnQqOjFLsrPXQ8uDautMbYHeeU0IAVhc6vGVla3HFdKtA6E0hFtd31gqXxMiQJAatfbSVrcnkkkCod02GLiJgdgT+5Z/TYI1dOS2vryYhdXfw9bmHAuJMiW1OyLXFugmRXb7DypH9YfNBaJg1CqoSaW3cRY5DdQFL7jxMCRUGPh6rEBaib1aW2F2EM2iUrtAH37jzH8zE/nJNBER3xyfie55n1eWUyRE5PGIhvytt9riLfLGc8BhTMhIUc1APmVAeg2/5frZCkaSZvcUEoRGHQ+/vJWbCCxcZq5VJvJeeuaoBCmUB/JnyWEgsbMt0lpqACPok/Ck6MzTF3jk/YizDWcWyEgLOgfMTF4xfcB3c= ceph-a70c765e-740d-11ef-bceb-c7b262605968 2024-09-16T09:26:50.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:50 smithi078 ceph-mon[23694]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:26:50.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:50 smithi078 ceph-mon[23694]: Generating ssh key... 2024-09-16T09:26:50.642 INFO:teuthology.orchestra.run.smithi078.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-16T09:26:50.642 INFO:teuthology.orchestra.run.smithi078.stderr:Adding key to root@localhost authorized_keys... 2024-09-16T09:26:50.652 INFO:teuthology.orchestra.run.smithi078.stderr:Adding host smithi078... 2024-09-16T09:26:51.521 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:51 smithi078 ceph-mon[23694]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:26:52.120 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Added host 'smithi078' with addr '172.21.15.78' 2024-09-16T09:26:52.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:52 smithi078 ceph-mon[23694]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi078", "addr": "172.21.15.78", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:26:52.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:52 smithi078 ceph-mon[23694]: from='mgr.14118 172.21.15.78:0/3051481559' entity='mgr.smithi078.rrfxge' 2024-09-16T09:26:52.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:52 smithi078 ceph-mon[23694]: from='mgr.14118 172.21.15.78:0/3051481559' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:26:52.606 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying mon service with default placement... 2024-09-16T09:26:53.218 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled mon update... 2024-09-16T09:26:53.529 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:53 smithi078 ceph-mon[23694]: Deploying cephadm binary to smithi078 2024-09-16T09:26:53.529 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:53 smithi078 ceph-mon[23694]: Added host smithi078 2024-09-16T09:26:53.529 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:53 smithi078 ceph-mon[23694]: from='mgr.14118 172.21.15.78:0/3051481559' entity='mgr.smithi078.rrfxge' 2024-09-16T09:26:53.989 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying mgr service with default placement... 2024-09-16T09:26:54.527 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:54 smithi078 ceph-mon[23694]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:26:54.527 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:54 smithi078 ceph-mon[23694]: Saving service mon spec with placement count:5 2024-09-16T09:26:54.613 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-09-16T09:26:55.457 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying crash service with default placement... 2024-09-16T09:26:55.866 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:55 smithi078 ceph-mon[23694]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:26:55.867 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:55 smithi078 ceph-mon[23694]: Saving service mgr spec with placement count:2 2024-09-16T09:26:55.867 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:55 smithi078 ceph-mon[23694]: from='mgr.14118 172.21.15.78:0/3051481559' entity='mgr.smithi078.rrfxge' 2024-09-16T09:26:56.083 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled crash update... 2024-09-16T09:26:57.016 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying prometheus service with default placement... 2024-09-16T09:26:57.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:57 smithi078 ceph-mon[23694]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:26:57.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:57 smithi078 ceph-mon[23694]: Saving service crash spec with placement * 2024-09-16T09:26:57.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:57 smithi078 ceph-mon[23694]: from='mgr.14118 172.21.15.78:0/3051481559' entity='mgr.smithi078.rrfxge' 2024-09-16T09:26:57.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:57 smithi078 ceph-mon[23694]: from='mgr.14118 172.21.15.78:0/3051481559' entity='mgr.smithi078.rrfxge' 2024-09-16T09:26:57.624 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-09-16T09:26:58.215 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying grafana service with default placement... 2024-09-16T09:26:58.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:58 smithi078 ceph-mon[23694]: from='mgr.14118 172.21.15.78:0/3051481559' entity='mgr.smithi078.rrfxge' 2024-09-16T09:26:58.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:58 smithi078 ceph-mon[23694]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:26:58.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:58 smithi078 ceph-mon[23694]: Saving service prometheus spec with placement count:1 2024-09-16T09:26:58.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:58 smithi078 ceph-mon[23694]: from='mgr.14118 172.21.15.78:0/3051481559' entity='mgr.smithi078.rrfxge' 2024-09-16T09:26:58.852 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-09-16T09:26:59.489 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying node-exporter service with default placement... 2024-09-16T09:27:00.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:59 smithi078 ceph-mon[23694]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:27:00.046 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:59 smithi078 ceph-mon[23694]: Saving service grafana spec with placement count:1 2024-09-16T09:27:00.046 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:26:59 smithi078 ceph-mon[23694]: from='mgr.14118 172.21.15.78:0/3051481559' entity='mgr.smithi078.rrfxge' 2024-09-16T09:27:00.486 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-09-16T09:27:01.064 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying alertmanager service with default placement... 2024-09-16T09:27:01.313 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:01 smithi078 ceph-mon[23694]: from='mgr.14118 172.21.15.78:0/3051481559' entity='mgr.smithi078.rrfxge' 2024-09-16T09:27:01.313 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:01 smithi078 ceph-mon[23694]: from='mgr.14118 172.21.15.78:0/3051481559' entity='mgr.smithi078.rrfxge' 2024-09-16T09:27:01.314 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:01 smithi078 ceph-mon[23694]: from='mgr.14118 172.21.15.78:0/3051481559' entity='mgr.smithi078.rrfxge' 2024-09-16T09:27:01.710 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-09-16T09:27:02.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:02 smithi078 ceph-mon[23694]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:27:02.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:02 smithi078 ceph-mon[23694]: Saving service node-exporter spec with placement * 2024-09-16T09:27:02.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:02 smithi078 ceph-mon[23694]: from='mgr.14118 172.21.15.78:0/3051481559' entity='mgr.smithi078.rrfxge' 2024-09-16T09:27:03.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:03 smithi078 ceph-mon[23694]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:27:03.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:03 smithi078 ceph-mon[23694]: Saving service alertmanager spec with placement count:1 2024-09-16T09:27:05.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:04 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/436095466' entity='client.admin' 2024-09-16T09:27:05.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:04 smithi078 ceph-mon[23694]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:27:05.457 INFO:teuthology.orchestra.run.smithi078.stderr:Enabling the dashboard module... 2024-09-16T09:27:06.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:05 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/1949149318' entity='client.admin' 2024-09-16T09:27:07.185 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:06 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/3152053796' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-16T09:27:07.186 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:06 smithi078 ceph-mon[23694]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:27:08.054 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:07 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/3152053796' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-16T09:27:08.054 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:07 smithi078 ceph-mon[23694]: mgrmap e9: smithi078.rrfxge(active, since 23s) 2024-09-16T09:27:08.210 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-09-16T09:27:08.210 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 9, 2024-09-16T09:27:08.210 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": true, 2024-09-16T09:27:08.210 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "active_name": "smithi078.rrfxge", 2024-09-16T09:27:08.211 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-16T09:27:08.211 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-09-16T09:27:08.845 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for the mgr to restart... 2024-09-16T09:27:08.845 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mgr epoch 9... 2024-09-16T09:27:08.968 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:08 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/1592148946' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-16T09:27:14.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:14 smithi078 ceph-mon[23694]: Active manager daemon smithi078.rrfxge restarted 2024-09-16T09:27:14.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:14 smithi078 ceph-mon[23694]: Activating manager daemon smithi078.rrfxge 2024-09-16T09:27:14.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:14 smithi078 ceph-mon[23694]: osdmap e3: 0 total, 0 up, 0 in 2024-09-16T09:27:14.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:14 smithi078 ceph-mon[23694]: mgrmap e10: smithi078.rrfxge(active, starting, since 0.055238s) 2024-09-16T09:27:14.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:14 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-16T09:27:14.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:14 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mgr metadata", "who": "smithi078.rrfxge", "id": "smithi078.rrfxge"}]: dispatch 2024-09-16T09:27:14.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:14 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-16T09:27:14.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:14 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-16T09:27:14.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:14 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-16T09:27:14.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:14 smithi078 ceph-mon[23694]: Manager daemon smithi078.rrfxge is now available 2024-09-16T09:27:14.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:14 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:27:14.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:14 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:27:14.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:14 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:27:14.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:14 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/trash_purge_schedule"}]: dispatch 2024-09-16T09:27:14.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:14 smithi078 ceph-mon[23694]: [16/Sep/2024:09:27:13] ENGINE Bus STARTING 2024-09-16T09:27:14.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:14 smithi078 ceph-mon[23694]: [16/Sep/2024:09:27:13] ENGINE Serving on https://172.21.15.78:7150 2024-09-16T09:27:14.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:14 smithi078 ceph-mon[23694]: [16/Sep/2024:09:27:13] ENGINE Bus STARTED 2024-09-16T09:27:14.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:14 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' 2024-09-16T09:27:14.075 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-09-16T09:27:14.075 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-09-16T09:27:14.075 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "initialized": true 2024-09-16T09:27:14.076 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-09-16T09:27:14.687 INFO:teuthology.orchestra.run.smithi078.stderr:mgr epoch 9 is available 2024-09-16T09:27:14.687 INFO:teuthology.orchestra.run.smithi078.stderr:Generating a dashboard self-signed certificate... 2024-09-16T09:27:15.250 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:15 smithi078 ceph-mon[23694]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-16T09:27:15.251 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:15 smithi078 ceph-mon[23694]: mgrmap e11: smithi078.rrfxge(active, since 1.05735s) 2024-09-16T09:27:15.251 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:15 smithi078 ceph-mon[23694]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-16T09:27:15.251 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:15 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' 2024-09-16T09:27:15.563 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Self-signed certificate created 2024-09-16T09:27:16.169 INFO:teuthology.orchestra.run.smithi078.stderr:Creating initial admin user... 2024-09-16T09:27:16.754 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:16 smithi078 ceph-mon[23694]: from='client.14172 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:27:16.754 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:16 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' 2024-09-16T09:27:16.754 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:16 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' 2024-09-16T09:27:16.754 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:16 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' 2024-09-16T09:27:16.755 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:16 smithi078 ceph-mon[23694]: mgrmap e12: smithi078.rrfxge(active, since 2s) 2024-09-16T09:27:17.013 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$HemEBlEN8cVChldeVo8KLO6jPdRWTKOIZEirx1oqEbGuFmhUyQ8ii", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726478837, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-16T09:27:17.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:17 smithi078 ceph-mon[23694]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:27:17.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:17 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' 2024-09-16T09:27:18.449 INFO:teuthology.orchestra.run.smithi078.stderr:Fetching dashboard port number... 2024-09-16T09:27:19.139 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 8443 2024-09-16T09:27:19.410 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:19 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/2896852018' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-16T09:27:19.747 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: disabled 2024-09-16T09:27:19.754 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2024-09-16T09:27:19.754 INFO:teuthology.orchestra.run.smithi078.stderr:firewalld.service is not enabled 2024-09-16T09:27:19.754 INFO:teuthology.orchestra.run.smithi078.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-16T09:27:19.756 INFO:teuthology.orchestra.run.smithi078.stderr:Ceph Dashboard is now available at: 2024-09-16T09:27:19.756 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-16T09:27:19.756 INFO:teuthology.orchestra.run.smithi078.stderr: URL: https://smithi078.front.sepia.ceph.com:8443/ 2024-09-16T09:27:19.756 INFO:teuthology.orchestra.run.smithi078.stderr: User: admin 2024-09-16T09:27:19.757 INFO:teuthology.orchestra.run.smithi078.stderr: Password: y7og0vadrk 2024-09-16T09:27:19.757 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-16T09:27:19.757 INFO:teuthology.orchestra.run.smithi078.stderr:Enabling autotune for osd_memory_target 2024-09-16T09:27:20.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:20 smithi078 ceph-mon[23694]: mgrmap e13: smithi078.rrfxge(active, since 6s) 2024-09-16T09:27:21.716 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-09-16T09:27:22.351 INFO:teuthology.orchestra.run.smithi078.stderr:You can access the Ceph CLI with: 2024-09-16T09:27:22.352 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-16T09:27:22.352 INFO:teuthology.orchestra.run.smithi078.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid a70c765e-740d-11ef-bceb-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-16T09:27:22.352 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-16T09:27:22.352 INFO:teuthology.orchestra.run.smithi078.stderr:Please consider enabling telemetry to help improve Ceph: 2024-09-16T09:27:22.352 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-16T09:27:22.352 INFO:teuthology.orchestra.run.smithi078.stderr: ceph telemetry on 2024-09-16T09:27:22.352 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-16T09:27:22.352 INFO:teuthology.orchestra.run.smithi078.stderr:For more information see: 2024-09-16T09:27:22.352 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-16T09:27:22.352 INFO:teuthology.orchestra.run.smithi078.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-16T09:27:22.352 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-16T09:27:22.352 INFO:teuthology.orchestra.run.smithi078.stderr:Bootstrap complete. 2024-09-16T09:27:22.390 INFO:tasks.cephadm:Fetching config... 2024-09-16T09:27:22.390 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-16T09:27:22.390 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-16T09:27:22.408 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-16T09:27:22.408 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-16T09:27:22.408 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-16T09:27:22.465 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-16T09:27:22.466 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-16T09:27:22.466 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/mon.smithi078/keyring of=/dev/stdout 2024-09-16T09:27:22.529 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-16T09:27:22.530 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-16T09:27:22.530 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-16T09:27:22.585 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-16T09:27:22.586 DEBUG:teuthology.orchestra.run.smithi078:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDnRbVwS/ERhvRSm6fncziAziD85QZvoTIWYudopUdrbEOyGXYEavCndGoTX+FTrrvgeg9Qszr0HvrtgHi5o9hpzPlLHLI4R8Q0hTss1yC1lC95BnQGWVb8Ya7z03dLJfQGsSkeFPdMv2U8xkGTdEp4btKnQqOjFLsrPXQ8uDautMbYHeeU0IAVhc6vGVla3HFdKtA6E0hFtd31gqXxMiQJAatfbSVrcnkkkCod02GLiJgdgT+5Z/TYI1dOS2vryYhdXfw9bmHAuJMiW1OyLXFugmRXb7DypH9YfNBaJg1CqoSaW3cRY5DdQFL7jxMCRUGPh6rEBaib1aW2F2EM2iUrtAH37jzH8zE/nJNBER3xyfie55n1eWUyRE5PGIhvytt9riLfLGc8BhTMhIUc1APmVAeg2/5frZCkaSZvcUEoRGHQ+/vJWbCCxcZq5VJvJeeuaoBCmUB/JnyWEgsbMt0lpqACPok/Ck6MzTF3jk/YizDWcWyEgLOgfMTF4xfcB3c= ceph-a70c765e-740d-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-16T09:27:22.660 INFO:teuthology.orchestra.run.smithi078.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDnRbVwS/ERhvRSm6fncziAziD85QZvoTIWYudopUdrbEOyGXYEavCndGoTX+FTrrvgeg9Qszr0HvrtgHi5o9hpzPlLHLI4R8Q0hTss1yC1lC95BnQGWVb8Ya7z03dLJfQGsSkeFPdMv2U8xkGTdEp4btKnQqOjFLsrPXQ8uDautMbYHeeU0IAVhc6vGVla3HFdKtA6E0hFtd31gqXxMiQJAatfbSVrcnkkkCod02GLiJgdgT+5Z/TYI1dOS2vryYhdXfw9bmHAuJMiW1OyLXFugmRXb7DypH9YfNBaJg1CqoSaW3cRY5DdQFL7jxMCRUGPh6rEBaib1aW2F2EM2iUrtAH37jzH8zE/nJNBER3xyfie55n1eWUyRE5PGIhvytt9riLfLGc8BhTMhIUc1APmVAeg2/5frZCkaSZvcUEoRGHQ+/vJWbCCxcZq5VJvJeeuaoBCmUB/JnyWEgsbMt0lpqACPok/Ck6MzTF3jk/YizDWcWyEgLOgfMTF4xfcB3c= ceph-a70c765e-740d-11ef-bceb-c7b262605968 2024-09-16T09:27:22.671 DEBUG:teuthology.orchestra.run.smithi088:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDnRbVwS/ERhvRSm6fncziAziD85QZvoTIWYudopUdrbEOyGXYEavCndGoTX+FTrrvgeg9Qszr0HvrtgHi5o9hpzPlLHLI4R8Q0hTss1yC1lC95BnQGWVb8Ya7z03dLJfQGsSkeFPdMv2U8xkGTdEp4btKnQqOjFLsrPXQ8uDautMbYHeeU0IAVhc6vGVla3HFdKtA6E0hFtd31gqXxMiQJAatfbSVrcnkkkCod02GLiJgdgT+5Z/TYI1dOS2vryYhdXfw9bmHAuJMiW1OyLXFugmRXb7DypH9YfNBaJg1CqoSaW3cRY5DdQFL7jxMCRUGPh6rEBaib1aW2F2EM2iUrtAH37jzH8zE/nJNBER3xyfie55n1eWUyRE5PGIhvytt9riLfLGc8BhTMhIUc1APmVAeg2/5frZCkaSZvcUEoRGHQ+/vJWbCCxcZq5VJvJeeuaoBCmUB/JnyWEgsbMt0lpqACPok/Ck6MzTF3jk/YizDWcWyEgLOgfMTF4xfcB3c= ceph-a70c765e-740d-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-16T09:27:22.721 INFO:teuthology.orchestra.run.smithi088.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDnRbVwS/ERhvRSm6fncziAziD85QZvoTIWYudopUdrbEOyGXYEavCndGoTX+FTrrvgeg9Qszr0HvrtgHi5o9hpzPlLHLI4R8Q0hTss1yC1lC95BnQGWVb8Ya7z03dLJfQGsSkeFPdMv2U8xkGTdEp4btKnQqOjFLsrPXQ8uDautMbYHeeU0IAVhc6vGVla3HFdKtA6E0hFtd31gqXxMiQJAatfbSVrcnkkkCod02GLiJgdgT+5Z/TYI1dOS2vryYhdXfw9bmHAuJMiW1OyLXFugmRXb7DypH9YfNBaJg1CqoSaW3cRY5DdQFL7jxMCRUGPh6rEBaib1aW2F2EM2iUrtAH37jzH8zE/nJNBER3xyfie55n1eWUyRE5PGIhvytt9riLfLGc8BhTMhIUc1APmVAeg2/5frZCkaSZvcUEoRGHQ+/vJWbCCxcZq5VJvJeeuaoBCmUB/JnyWEgsbMt0lpqACPok/Ck6MzTF3jk/YizDWcWyEgLOgfMTF4xfcB3c= ceph-a70c765e-740d-11ef-bceb-c7b262605968 2024-09-16T09:27:22.733 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-16T09:27:22.878 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:22 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/3934498278' entity='client.admin' 2024-09-16T09:27:26.965 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-16T09:27:26.965 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-16T09:27:27.267 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:26 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/1635063038' entity='client.admin' 2024-09-16T09:27:27.267 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:26 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' 2024-09-16T09:27:27.267 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:26 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd/host:smithi078", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:27:27.267 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:26 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' 2024-09-16T09:27:28.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:27 smithi078 ceph-mon[23694]: Deploying daemon alertmanager.smithi078 on smithi078 2024-09-16T09:27:30.601 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi088 2024-09-16T09:27:30.602 DEBUG:teuthology.orchestra.run.smithi088:> set -ex 2024-09-16T09:27:30.602 DEBUG:teuthology.orchestra.run.smithi088:> dd of=/etc/ceph/ceph.conf 2024-09-16T09:27:30.621 DEBUG:teuthology.orchestra.run.smithi088:> set -ex 2024-09-16T09:27:30.621 DEBUG:teuthology.orchestra.run.smithi088:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-16T09:27:30.679 INFO:tasks.cephadm:Adding host smithi088 to orchestrator... 2024-09-16T09:27:30.679 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph orch host add smithi088 2024-09-16T09:27:30.703 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:30 smithi078 ceph-mon[23694]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:27:30.703 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:30 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' 2024-09-16T09:27:34.339 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:34 smithi078 ceph-mon[23694]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:27:34.339 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:34 smithi078 ceph-mon[23694]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi088", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:27:34.478 INFO:teuthology.orchestra.run.smithi078.stdout:Added host 'smithi088' with addr '172.21.15.88' 2024-09-16T09:27:35.125 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph orch host ls --format=json 2024-09-16T09:27:35.787 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:35 smithi078 ceph-mon[23694]: Deploying cephadm binary to smithi088 2024-09-16T09:27:35.787 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:35 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' 2024-09-16T09:27:35.787 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:35 smithi078 ceph-mon[23694]: Added host smithi088 2024-09-16T09:27:36.777 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:36 smithi078 ceph-mon[23694]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:27:37.452 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-16T09:27:37.453 INFO:teuthology.orchestra.run.smithi078.stdout:[{"addr": "172.21.15.78", "hostname": "smithi078", "labels": [], "status": ""}, {"addr": "172.21.15.88", "hostname": "smithi088", "labels": [], "status": ""}] 2024-09-16T09:27:38.428 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-16T09:27:38.428 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph osd crush tunables default 2024-09-16T09:27:38.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:38 smithi078 ceph-mon[23694]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:27:38.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:38 smithi078 ceph-mon[23694]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T09:27:38.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:38 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' 2024-09-16T09:27:38.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:38 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-16T09:27:38.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:38 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-16T09:27:38.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:38 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:27:39.510 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:39 smithi078 ceph-mon[23694]: Deploying daemon crash.smithi078 on smithi078 2024-09-16T09:27:40.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:40 smithi078 ceph-mon[23694]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:27:41.493 INFO:teuthology.orchestra.run.smithi078.stderr:adjusted tunables profile to default 2024-09-16T09:27:41.795 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:41 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/3549654452' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-16T09:27:42.299 INFO:tasks.cephadm:Adding mon.smithi078 on smithi078 2024-09-16T09:27:42.299 INFO:tasks.cephadm:Adding mon.smithi088 on smithi088 2024-09-16T09:27:42.300 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph orch apply mon '2;smithi078:172.21.15.78=smithi078;smithi088:172.21.15.88=smithi088' 2024-09-16T09:27:42.488 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:42 smithi078 ceph-mon[23694]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:27:42.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:42 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/3549654452' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-16T09:27:42.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:42 smithi078 ceph-mon[23694]: osdmap e4: 0 total, 0 up, 0 in 2024-09-16T09:27:42.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:42 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' 2024-09-16T09:27:42.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:42 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' 2024-09-16T09:27:42.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:42 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' 2024-09-16T09:27:42.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:42 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-16T09:27:42.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:42 smithi078 ceph-mon[23694]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-16T09:27:42.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:42 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' 2024-09-16T09:27:42.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:42 smithi078 ceph-mon[23694]: Deploying daemon grafana.smithi078 on smithi078 2024-09-16T09:27:44.050 INFO:teuthology.orchestra.run.smithi088.stdout:Scheduled mon update... 2024-09-16T09:27:44.555 DEBUG:teuthology.orchestra.run.smithi088:mon.smithi088> sudo journalctl -f -n 0 -u ceph-a70c765e-740d-11ef-bceb-c7b262605968@mon.smithi088.service 2024-09-16T09:27:44.557 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:27:44.558 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:27:44.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:44 smithi078 ceph-mon[23694]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:27:44.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:44 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' 2024-09-16T09:27:44.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:44 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' 2024-09-16T09:27:45.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:45 smithi078 ceph-mon[23694]: from='client.14192 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi078:172.21.15.78=smithi078;smithi088:172.21.15.88=smithi088", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:27:45.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:45 smithi078 ceph-mon[23694]: Saving service mon spec with placement smithi078:172.21.15.78=smithi078;smithi088:172.21.15.88=smithi088;count:2 2024-09-16T09:27:46.394 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:27:46.394 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:27:46.396 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:27:46.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:46 smithi078 ceph-mon[23694]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:27:47.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:47 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/2566040165' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:27:47.997 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:27:47.997 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:27:48.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:48 smithi078 ceph-mon[23694]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:27:49.671 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:27:49.671 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:27:49.673 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:27:50.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:50 smithi078 ceph-mon[23694]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:27:50.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:50 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/719052903' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:27:51.250 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:27:51.250 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:27:52.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:52 smithi078 ceph-mon[23694]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:27:52.936 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:27:52.937 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:27:52.938 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:27:53.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:53 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/4085286450' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:27:54.512 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:27:54.512 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:27:54.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:54 smithi078 ceph-mon[23694]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:27:56.311 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:27:56.311 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:27:56.313 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:27:56.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:56 smithi078 ceph-mon[23694]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:27:56.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:56 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/1610255991' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:27:57.870 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:27:57.870 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:27:58.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:27:58 smithi078 ceph-mon[23694]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:27:59.678 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:27:59.678 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:27:59.680 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:28:00.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:00 smithi078 ceph-mon[23694]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:28:00.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:00 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/3883909602' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:28:01.256 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:28:01.256 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:28:02.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:02 smithi078 ceph-mon[23694]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:28:02.973 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:28:02.973 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:28:02.975 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:28:03.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:03 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/1481198219' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:28:04.510 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:28:04.510 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:28:04.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:04 smithi078 ceph-mon[23694]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:28:06.198 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:28:06.198 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:28:06.199 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:28:06.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:06 smithi078 ceph-mon[23694]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:28:06.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:06 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/1696517954' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:28:07.736 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:28:07.736 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:28:08.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:08 smithi078 ceph-mon[23694]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:28:09.494 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:28:09.494 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:28:09.496 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:28:10.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:10 smithi078 ceph-mon[23694]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:28:10.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:10 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/3055181772' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:28:11.081 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:28:11.081 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:28:12.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:12 smithi078 ceph-mon[23694]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:28:12.865 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:28:12.865 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:28:12.867 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:28:13.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:13 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/2146174989' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:28:13.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:13 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:28:13.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:13 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/trash_purge_schedule"}]: dispatch 2024-09-16T09:28:14.399 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:28:14.399 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:28:14.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:14 smithi078 ceph-mon[23694]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:28:16.276 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:28:16.276 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:28:16.279 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:28:16.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:16 smithi078 ceph-mon[23694]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:28:16.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:16 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/275939904' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:28:17.812 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:28:17.812 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:28:18.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:18 smithi078 ceph-mon[23694]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:28:19.603 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:28:19.604 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:28:19.605 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:28:20.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:20 smithi078 ceph-mon[23694]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:28:20.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:20 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/3302779771' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:28:21.190 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:28:21.190 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:28:22.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:22 smithi078 ceph-mon[23694]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:28:22.909 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:28:22.909 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:28:22.911 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:28:23.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:23 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/2651242898' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:28:24.514 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:28:24.514 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:28:24.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:24 smithi078 ceph-mon[23694]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:28:26.266 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:28:26.267 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:28:26.268 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:28:26.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:26 smithi078 ceph-mon[23694]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:28:26.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:26 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/1579818662' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:28:27.816 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:28:27.816 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:28:28.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:28 smithi078 ceph-mon[23694]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:28:29.441 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:28:29.441 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:28:29.442 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:28:30.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:30 smithi078 ceph-mon[23694]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:28:30.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:30 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/195071107' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:28:31.027 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:28:31.027 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:28:32.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:32 smithi078 ceph-mon[23694]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:28:32.848 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:28:32.848 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:28:32.850 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:28:33.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:33 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/3888646091' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:28:34.397 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:28:34.397 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:28:34.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:34 smithi078 ceph-mon[23694]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:28:36.174 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:28:36.174 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:28:36.176 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:28:36.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:36 smithi078 ceph-mon[23694]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:28:36.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:36 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/3028169631' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:28:37.614 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:28:37.614 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:28:38.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:38 smithi078 ceph-mon[23694]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:28:39.417 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:28:39.418 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:28:39.420 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:28:40.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:40 smithi078 ceph-mon[23694]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:28:40.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:40 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/3421874826' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:28:40.962 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:28:40.963 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:28:42.608 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:28:42.608 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:28:42.610 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:28:42.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:42 smithi078 ceph-mon[23694]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:28:43.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:43 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/829629425' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:28:44.209 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:28:44.210 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:28:44.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:44 smithi078 ceph-mon[23694]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:28:46.022 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:28:46.023 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:28:46.024 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:28:46.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:46 smithi078 ceph-mon[23694]: pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:28:46.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:46 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/3618104573' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:28:47.585 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:28:47.585 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:28:48.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:48 smithi078 ceph-mon[23694]: pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:28:49.414 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:28:49.415 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:28:49.417 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:28:50.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:50 smithi078 ceph-mon[23694]: pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:28:50.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:50 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/16504895' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:28:50.988 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:28:50.989 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:28:52.627 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:28:52.627 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:28:52.629 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:28:52.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:52 smithi078 ceph-mon[23694]: pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:28:53.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:53 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/2370671737' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:28:54.241 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:28:54.242 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:28:54.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:54 smithi078 ceph-mon[23694]: pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:28:55.934 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:28:55.934 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:28:55.936 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:28:56.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:56 smithi078 ceph-mon[23694]: pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:28:56.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:56 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/138536167' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:28:57.530 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:28:57.531 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:28:58.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:58 smithi078 ceph-mon[23694]: pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:28:59.216 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:28:59.216 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:28:59.218 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:28:59.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:28:59 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/740993567' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:29:00.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:00 smithi078 ceph-mon[23694]: pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:00.815 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:29:00.816 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:29:02.603 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:29:02.603 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:29:02.605 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:29:02.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:02 smithi078 ceph-mon[23694]: pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:03.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:03 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/417274899' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:29:04.194 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:29:04.195 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:29:04.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:04 smithi078 ceph-mon[23694]: pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:05.978 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:29:05.978 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:29:05.979 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:29:06.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:06 smithi078 ceph-mon[23694]: pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:06.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:06 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/1934296928' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:29:07.534 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:29:07.534 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:29:08.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:08 smithi078 ceph-mon[23694]: pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:09.369 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:29:09.369 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:29:09.370 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:29:09.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:09 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/2752310417' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:29:10.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:10 smithi078 ceph-mon[23694]: pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:10.885 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:29:10.885 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:29:12.588 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:29:12.588 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:29:12.590 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:29:12.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:12 smithi078 ceph-mon[23694]: pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:13.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:13 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/3752828360' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:29:13.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:13 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:29:13.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:13 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/trash_purge_schedule"}]: dispatch 2024-09-16T09:29:14.130 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:29:14.131 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:29:14.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:14 smithi078 ceph-mon[23694]: pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:15.964 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:29:15.965 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:29:15.966 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:29:16.743 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:16 smithi078 ceph-mon[23694]: pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:16.743 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:16 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/1952828841' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:29:17.533 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:29:17.533 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:29:18.505 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:18 smithi078 ceph-mon[23694]: pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:19.457 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:29:19.457 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:29:19.459 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:29:19.744 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:19 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:19.744 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:19 smithi078 ceph-mon[23694]: Deploying daemon node-exporter.smithi078 on smithi078 2024-09-16T09:29:19.745 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:19 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/402764849' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:29:20.675 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:20 smithi078 ceph-mon[23694]: pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:21.055 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:29:21.055 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:29:22.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:22 smithi078 ceph-mon[23694]: pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:22.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:22 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:22.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:22 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-16T09:29:22.926 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:29:22.926 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:29:22.928 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:29:23.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:23 smithi078 ceph-mon[23694]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-16T09:29:23.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:23 smithi078 ceph-mon[23694]: mgrmap e14: smithi078.rrfxge(active, since 2m) 2024-09-16T09:29:23.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:23 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/2316460971' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:29:24.507 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:29:24.507 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:29:26.437 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:29:26.437 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:29:26.439 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:29:26.779 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:26 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/2320471094' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:29:28.027 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:29:28.027 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:29:29.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:29 smithi078 ceph-mon[23694]: Active manager daemon smithi078.rrfxge restarted 2024-09-16T09:29:29.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:29 smithi078 ceph-mon[23694]: Activating manager daemon smithi078.rrfxge 2024-09-16T09:29:29.620 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:29 smithi078 ceph-mon[23694]: osdmap e5: 0 total, 0 up, 0 in 2024-09-16T09:29:30.029 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:29:30.029 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:29:30.031 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:29:30.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:30 smithi078 ceph-mon[23694]: mgrmap e15: smithi078.rrfxge(active, starting, since 0.806282s) 2024-09-16T09:29:30.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:30 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-16T09:29:30.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:30 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mgr metadata", "who": "smithi078.rrfxge", "id": "smithi078.rrfxge"}]: dispatch 2024-09-16T09:29:30.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:30 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-16T09:29:30.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:30 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-16T09:29:30.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:30 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-16T09:29:30.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:30 smithi078 ceph-mon[23694]: Manager daemon smithi078.rrfxge is now available 2024-09-16T09:29:30.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:30 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:29:30.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:30 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:29:30.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:30 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:29:30.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:30 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:29:30.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:30 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:30.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:30 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:29:30.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:30 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/trash_purge_schedule"}]: dispatch 2024-09-16T09:29:30.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:30 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:30.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:30 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/4145372374' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:29:31.669 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:29:31.670 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:29:31.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:31 smithi078 ceph-mon[23694]: mgrmap e16: smithi078.rrfxge(active, since 1.80828s) 2024-09-16T09:29:31.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:31 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:31.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:31 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:32.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:32 smithi078 ceph-mon[23694]: [16/Sep/2024:09:29:30] ENGINE Bus STARTING 2024-09-16T09:29:32.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:32 smithi078 ceph-mon[23694]: [16/Sep/2024:09:29:30] ENGINE Serving on https://172.21.15.78:7150 2024-09-16T09:29:32.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:32 smithi078 ceph-mon[23694]: [16/Sep/2024:09:29:30] ENGINE Bus STARTED 2024-09-16T09:29:32.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:32 smithi078 ceph-mon[23694]: mgrmap e17: smithi078.rrfxge(active, since 2s) 2024-09-16T09:29:32.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:32 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:35.483 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:29:35.483 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:29:35.485 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:29:35.702 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:35 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:35.702 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:35 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:35.702 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:35 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd/host:smithi088", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:29:35.703 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:35 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:36.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:36 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/2954457276' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:29:36.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:36 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:36.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:36 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd/host:smithi078", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:29:36.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:36 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:36.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:36 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:36.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:36 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi088", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-16T09:29:36.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:36 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi088", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-16T09:29:36.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:36 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:29:37.064 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:29:37.064 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:29:37.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:37 smithi078 ceph-mon[23694]: Updating smithi078:/etc/ceph/ceph.conf 2024-09-16T09:29:37.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:37 smithi078 ceph-mon[23694]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-09-16T09:29:37.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:37 smithi078 ceph-mon[23694]: Deploying daemon crash.smithi088 on smithi088 2024-09-16T09:29:39.654 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:29:39.654 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:29:39.656 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:29:40.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:39 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/1337184518' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:29:41.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:41 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:41.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:41 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi088.ncxfdz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T09:29:41.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:41 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi088.ncxfdz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-16T09:29:41.312 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:41 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-16T09:29:41.312 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:41 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:29:41.312 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:41 smithi078 ceph-mon[23694]: Deploying daemon mgr.smithi088.ncxfdz on smithi088 2024-09-16T09:29:41.413 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:29:41.414 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:29:43.401 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:29:43.401 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:29:43.403 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:29:44.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:44 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/2902094977' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:29:44.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:44 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:44.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:44 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-16T09:29:44.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:44 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:29:44.979 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:29:44.980 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:29:45.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:45 smithi078 ceph-mon[23694]: Deploying daemon mon.smithi088 on smithi088 2024-09-16T09:29:45.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:45 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:47.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:47 smithi088 systemd[1]: Starting Ceph mon.smithi088 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:29:48.412 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 podman[29094]: 2024-09-16 09:29:48.355165144 +0000 UTC m=+0.342410078 container create 774a14c001137d19729dd3c5b34889d2f9899d5b6fd9349c7cfefee18320bd7c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi088, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, GIT_CLEAN=True, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, GIT_BRANCH=HEAD, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0) 2024-09-16T09:29:48.412 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 podman[29094]: 2024-09-16 09:29:48.260093558 +0000 UTC m=+0.247338505 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-09-16T09:29:48.480 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:29:48.480 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:26:06.341058Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T09:29:48.482 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-16T09:29:48.690 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 podman[29094]: 2024-09-16 09:29:48.58335649 +0000 UTC m=+0.570601433 container init 774a14c001137d19729dd3c5b34889d2f9899d5b6fd9349c7cfefee18320bd7c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi088, io.k8s.display-name=CentOS Stream 8, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, com.redhat.component=centos-stream-container, version=8, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, name=centos-stream, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, RELEASE=HEAD, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True) 2024-09-16T09:29:48.690 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 podman[29094]: 2024-09-16 09:29:48.587238709 +0000 UTC m=+0.574483651 container start 774a14c001137d19729dd3c5b34889d2f9899d5b6fd9349c7cfefee18320bd7c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi088, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-16T09:29:48.691 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: set uid:gid to 167:167 (ceph:ceph) 2024-09-16T09:29:48.691 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-16T09:29:48.691 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: pidfile_write: ignore empty --pid-file 2024-09-16T09:29:48.691 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: load: jerasure load: lrc 2024-09-16T09:29:48.691 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: RocksDB version: 6.15.5 2024-09-16T09:29:48.691 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-16T09:29:48.691 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Compile date Apr 18 2022 2024-09-16T09:29:48.691 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: DB SUMMARY 2024-09-16T09:29:48.692 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: DB Session ID: 8VSWL4QX4H97WU8BHGRM 2024-09-16T09:29:48.692 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: CURRENT file: CURRENT 2024-09-16T09:29:48.692 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: IDENTITY file: IDENTITY 2024-09-16T09:29:48.692 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-16T09:29:48.692 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi088/store.db dir, Total Num: 0, files: 2024-09-16T09:29:48.692 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi088/store.db: 000004.log size: 511 ; 2024-09-16T09:29:48.692 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.error_if_exists: 0 2024-09-16T09:29:48.692 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.create_if_missing: 0 2024-09-16T09:29:48.692 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.paranoid_checks: 1 2024-09-16T09:29:48.692 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-16T09:29:48.692 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.env: 0x557750ec6860 2024-09-16T09:29:48.692 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.fs: Posix File System 2024-09-16T09:29:48.693 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.info_log: 0x5577533e7fa0 2024-09-16T09:29:48.693 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.max_file_opening_threads: 16 2024-09-16T09:29:48.693 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.statistics: (nil) 2024-09-16T09:29:48.693 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.use_fsync: 0 2024-09-16T09:29:48.693 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.max_log_file_size: 0 2024-09-16T09:29:48.693 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-16T09:29:48.693 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-16T09:29:48.693 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.keep_log_file_num: 1000 2024-09-16T09:29:48.693 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.recycle_log_file_num: 0 2024-09-16T09:29:48.693 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.allow_fallocate: 1 2024-09-16T09:29:48.693 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.allow_mmap_reads: 0 2024-09-16T09:29:48.693 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.allow_mmap_writes: 0 2024-09-16T09:29:48.693 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.use_direct_reads: 0 2024-09-16T09:29:48.693 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-16T09:29:48.694 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.create_missing_column_families: 0 2024-09-16T09:29:48.694 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.db_log_dir: 2024-09-16T09:29:48.694 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi088/store.db 2024-09-16T09:29:48.694 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-16T09:29:48.694 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-16T09:29:48.694 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-16T09:29:48.694 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-16T09:29:48.694 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-16T09:29:48.694 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-16T09:29:48.695 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.advise_random_on_open: 1 2024-09-16T09:29:48.695 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.db_write_buffer_size: 0 2024-09-16T09:29:48.696 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.write_buffer_manager: 0x5577534d8330 2024-09-16T09:29:48.696 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-16T09:29:48.696 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-16T09:29:48.696 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-16T09:29:48.696 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-16T09:29:48.696 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.rate_limiter: (nil) 2024-09-16T09:29:48.696 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-16T09:29:48.696 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.wal_recovery_mode: 2 2024-09-16T09:29:48.696 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.enable_thread_tracking: 0 2024-09-16T09:29:48.696 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.enable_pipelined_write: 0 2024-09-16T09:29:48.696 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.unordered_write: 0 2024-09-16T09:29:48.696 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-16T09:29:48.696 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-16T09:29:48.697 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-16T09:29:48.697 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-16T09:29:48.697 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.row_cache: None 2024-09-16T09:29:48.697 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.wal_filter: None 2024-09-16T09:29:48.697 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-16T09:29:48.697 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.allow_ingest_behind: 0 2024-09-16T09:29:48.697 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.preserve_deletes: 0 2024-09-16T09:29:48.697 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.two_write_queues: 0 2024-09-16T09:29:48.697 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.manual_wal_flush: 0 2024-09-16T09:29:48.697 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.atomic_flush: 0 2024-09-16T09:29:48.697 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-16T09:29:48.698 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-16T09:29:48.698 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-16T09:29:48.698 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.log_readahead_size: 0 2024-09-16T09:29:48.698 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-16T09:29:48.698 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.best_efforts_recovery: 0 2024-09-16T09:29:48.698 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-16T09:29:48.698 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-16T09:29:48.698 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.allow_data_in_errors: 0 2024-09-16T09:29:48.698 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.db_host_id: __hostname__ 2024-09-16T09:29:48.698 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.max_background_jobs: 2 2024-09-16T09:29:48.698 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.max_background_compactions: -1 2024-09-16T09:29:48.698 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.max_subcompactions: 1 2024-09-16T09:29:48.698 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-16T09:29:48.698 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-16T09:29:48.699 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-16T09:29:48.699 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.max_total_wal_size: 0 2024-09-16T09:29:48.699 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-16T09:29:48.699 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-16T09:29:48.699 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-16T09:29:48.699 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-16T09:29:48.699 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.max_open_files: -1 2024-09-16T09:29:48.699 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.bytes_per_sync: 0 2024-09-16T09:29:48.699 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-16T09:29:48.699 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-16T09:29:48.699 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.compaction_readahead_size: 0 2024-09-16T09:29:48.699 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.max_background_flushes: -1 2024-09-16T09:29:48.699 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Compression algorithms supported: 2024-09-16T09:29:48.699 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-16T09:29:48.700 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: kZSTD supported: 0 2024-09-16T09:29:48.700 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: kXpressCompression supported: 0 2024-09-16T09:29:48.700 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: kLZ4HCCompression supported: 1 2024-09-16T09:29:48.700 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: kLZ4Compression supported: 1 2024-09-16T09:29:48.700 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: kBZip2Compression supported: 0 2024-09-16T09:29:48.700 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: kZlibCompression supported: 1 2024-09-16T09:29:48.700 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: kSnappyCompression supported: 1 2024-09-16T09:29:48.700 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-16T09:29:48.700 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi088/store.db/MANIFEST-000003 2024-09-16T09:29:48.700 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-16T09:29:48.700 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-16T09:29:48.700 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.merge_operator: 2024-09-16T09:29:48.700 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.compaction_filter: None 2024-09-16T09:29:48.700 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.compaction_filter_factory: None 2024-09-16T09:29:48.701 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.sst_partitioner_factory: None 2024-09-16T09:29:48.701 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-16T09:29:48.701 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-16T09:29:48.701 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5577533b3d00) 2024-09-16T09:29:48.701 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: cache_index_and_filter_blocks: 1 2024-09-16T09:29:48.701 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-16T09:29:48.701 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-16T09:29:48.701 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: pin_top_level_index_and_filter: 1 2024-09-16T09:29:48.701 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: index_type: 0 2024-09-16T09:29:48.701 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: data_block_index_type: 0 2024-09-16T09:29:48.701 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: index_shortening: 1 2024-09-16T09:29:48.701 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-16T09:29:48.701 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: hash_index_allow_collision: 1 2024-09-16T09:29:48.702 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: checksum: 1 2024-09-16T09:29:48.702 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: no_block_cache: 0 2024-09-16T09:29:48.702 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: block_cache: 0x55775341e170 2024-09-16T09:29:48.702 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: block_cache_name: BinnedLRUCache 2024-09-16T09:29:48.702 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: block_cache_options: 2024-09-16T09:29:48.702 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: capacity : 536870912 2024-09-16T09:29:48.702 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: num_shard_bits : 4 2024-09-16T09:29:48.702 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: strict_capacity_limit : 0 2024-09-16T09:29:48.702 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: high_pri_pool_ratio: 0.000 2024-09-16T09:29:48.702 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: block_cache_compressed: (nil) 2024-09-16T09:29:48.702 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: persistent_cache: (nil) 2024-09-16T09:29:48.702 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: block_size: 4096 2024-09-16T09:29:48.702 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: block_size_deviation: 10 2024-09-16T09:29:48.703 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: block_restart_interval: 16 2024-09-16T09:29:48.703 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: index_block_restart_interval: 1 2024-09-16T09:29:48.703 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: metadata_block_size: 4096 2024-09-16T09:29:48.703 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: partition_filters: 0 2024-09-16T09:29:48.703 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: use_delta_encoding: 1 2024-09-16T09:29:48.703 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-16T09:29:48.703 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: whole_key_filtering: 1 2024-09-16T09:29:48.703 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: verify_compression: 0 2024-09-16T09:29:48.703 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: read_amp_bytes_per_bit: 0 2024-09-16T09:29:48.703 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: format_version: 4 2024-09-16T09:29:48.703 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: enable_index_compression: 1 2024-09-16T09:29:48.703 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: block_align: 0 2024-09-16T09:29:48.703 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.write_buffer_size: 33554432 2024-09-16T09:29:48.703 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.max_write_buffer_number: 2 2024-09-16T09:29:48.704 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.compression: NoCompression 2024-09-16T09:29:48.704 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.bottommost_compression: Disabled 2024-09-16T09:29:48.704 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.prefix_extractor: nullptr 2024-09-16T09:29:48.704 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-16T09:29:48.704 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.num_levels: 7 2024-09-16T09:29:48.704 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-16T09:29:48.704 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-16T09:29:48.704 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-16T09:29:48.704 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-16T09:29:48.704 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-16T09:29:48.704 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-16T09:29:48.704 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-16T09:29:48.704 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-16T09:29:48.704 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-16T09:29:48.705 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-16T09:29:48.705 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-16T09:29:48.705 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.compression_opts.level: 32767 2024-09-16T09:29:48.705 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.compression_opts.strategy: 0 2024-09-16T09:29:48.705 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-16T09:29:48.705 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-16T09:29:48.705 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-16T09:29:48.705 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.compression_opts.enabled: false 2024-09-16T09:29:48.705 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-16T09:29:48.705 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-16T09:29:48.705 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-16T09:29:48.705 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.target_file_size_base: 67108864 2024-09-16T09:29:48.705 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-16T09:29:48.706 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-16T09:29:48.706 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-16T09:29:48.706 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-16T09:29:48.706 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-16T09:29:48.706 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-16T09:29:48.706 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-16T09:29:48.706 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-16T09:29:48.706 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-16T09:29:48.706 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-16T09:29:48.706 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-16T09:29:48.706 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-16T09:29:48.706 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-16T09:29:48.706 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.arena_block_size: 4194304 2024-09-16T09:29:48.706 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-16T09:29:48.707 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-16T09:29:48.707 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-16T09:29:48.707 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.disable_auto_compactions: 0 2024-09-16T09:29:48.707 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-16T09:29:48.707 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-16T09:29:48.707 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-16T09:29:48.707 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-16T09:29:48.707 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-16T09:29:48.707 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-16T09:29:48.707 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-16T09:29:48.707 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-16T09:29:48.707 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-16T09:29:48.707 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-16T09:29:48.707 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.table_properties_collectors: 2024-09-16T09:29:48.708 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.inplace_update_support: 0 2024-09-16T09:29:48.708 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-16T09:29:48.708 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-16T09:29:48.708 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-16T09:29:48.708 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-16T09:29:48.708 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.bloom_locality: 0 2024-09-16T09:29:48.708 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.max_successive_merges: 0 2024-09-16T09:29:48.708 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-16T09:29:48.708 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.paranoid_file_checks: 0 2024-09-16T09:29:48.708 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.force_consistency_checks: 1 2024-09-16T09:29:48.708 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.report_bg_io_stats: 0 2024-09-16T09:29:48.708 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.ttl: 2592000 2024-09-16T09:29:48.708 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-16T09:29:48.708 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.enable_blob_files: false 2024-09-16T09:29:48.709 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.min_blob_size: 0 2024-09-16T09:29:48.709 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.blob_file_size: 268435456 2024-09-16T09:29:48.709 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-16T09:29:48.709 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-16T09:29:48.709 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-16T09:29:48.709 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi088/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-16T09:29:48.709 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-09-16T09:29:48.709 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-09-16T09:29:48.709 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726478988613364, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-16T09:29:48.709 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-09-16T09:29:48.709 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-16T09:29:48.709 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726478988613659, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726478988, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "0e6afdfc-379d-48b8-a541-bc891acafc01", "db_session_id": "8VSWL4QX4H97WU8BHGRM"}} 2024-09-16T09:29:48.709 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-09-16T09:29:48.710 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726478988613972, "job": 1, "event": "recovery_finished"} 2024-09-16T09:29:48.710 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi088/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-16T09:29:48.710 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x557753404a80 2024-09-16T09:29:48.710 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: DB pointer 0x557753414000 2024-09-16T09:29:48.710 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-16T09:29:48.710 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-16T09:29:48.710 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: ** DB Stats ** 2024-09-16T09:29:48.710 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-16T09:29:48.710 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-16T09:29:48.710 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-16T09:29:48.710 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-16T09:29:48.710 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-16T09:29:48.710 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-16T09:29:48.710 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-16T09:29:48.711 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: 2024-09-16T09:29:48.711 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: ** Compaction Stats [default] ** 2024-09-16T09:29:48.711 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-16T09:29:48.711 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-16T09:29:48.711 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-09-16T09:29:48.711 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-09-16T09:29:48.711 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-09-16T09:29:48.711 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: 2024-09-16T09:29:48.711 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: ** Compaction Stats [default] ** 2024-09-16T09:29:48.711 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-16T09:29:48.711 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-16T09:29:48.711 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-09-16T09:29:48.711 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-16T09:29:48.711 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-16T09:29:48.712 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-16T09:29:48.712 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-16T09:29:48.712 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-16T09:29:48.712 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-16T09:29:48.712 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Cumulative compaction: 0.00 GB write, 0.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-16T09:29:48.712 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Interval compaction: 0.00 GB write, 0.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-16T09:29:48.712 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-16T09:29:48.712 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: 2024-09-16T09:29:48.712 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-16T09:29:48.712 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: 2024-09-16T09:29:48.712 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: ** Compaction Stats [default] ** 2024-09-16T09:29:48.712 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-16T09:29:48.712 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-16T09:29:48.713 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-09-16T09:29:48.713 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-09-16T09:29:48.713 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-16T09:29:48.713 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: 2024-09-16T09:29:48.713 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: ** Compaction Stats [default] ** 2024-09-16T09:29:48.713 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-16T09:29:48.713 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-16T09:29:48.713 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.1 0.00 0.00 1 0.000 0 0 2024-09-16T09:29:48.713 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-16T09:29:48.713 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-16T09:29:48.713 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-16T09:29:48.713 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-16T09:29:48.713 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-16T09:29:48.713 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-16T09:29:48.714 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Cumulative compaction: 0.00 GB write, 0.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-16T09:29:48.714 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-16T09:29:48.714 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-16T09:29:48.714 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: 2024-09-16T09:29:48.714 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-16T09:29:48.714 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: mon.smithi088 does not exist in monmap, will attempt to join an existing cluster 2024-09-16T09:29:48.714 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: using public_addr v2:172.21.15.88:0/0 -> [v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0] 2024-09-16T09:29:48.715 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: starting mon.smithi088 rank -1 at public addrs [v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0] at bind addrs [v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi088 fsid a70c765e-740d-11ef-bceb-c7b262605968 2024-09-16T09:29:48.715 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: mon.smithi088@-1(???) e0 preinit fsid a70c765e-740d-11ef-bceb-c7b262605968 2024-09-16T09:29:48.715 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: mon.smithi088@-1(synchronizing).mds e1 new map 2024-09-16T09:29:48.715 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: mon.smithi088@-1(synchronizing).mds e1 print_map 2024-09-16T09:29:48.715 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: e1 2024-09-16T09:29:48.715 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-16T09:29:48.715 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-16T09:29:48.715 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: legacy client fscid: -1 2024-09-16T09:29:48.715 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: 2024-09-16T09:29:48.715 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: No filesystems configured 2024-09-16T09:29:48.715 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: mon.smithi088@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-16T09:29:48.716 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: mon.smithi088@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-16T09:29:48.716 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: mon.smithi088@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-16T09:29:48.716 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: mon.smithi088@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-16T09:29:48.716 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: mon.smithi088@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-16T09:29:48.716 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: mon.smithi088@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-16T09:29:48.716 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: mon.smithi088@-1(synchronizing).osd e5 e5: 0 total, 0 up, 0 in 2024-09-16T09:29:48.716 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: mon.smithi088@-1(synchronizing).osd e5 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-16T09:29:48.716 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: mon.smithi088@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-09-16T09:29:48.716 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: mon.smithi088@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-09-16T09:29:48.716 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: mon.smithi088@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-09-16T09:29:48.716 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:48.716 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='client.? 172.21.15.88:0/740993567' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:29:48.716 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:48.716 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:48.717 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='client.? 172.21.15.88:0/417274899' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:29:48.717 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:48.717 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:48.717 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='client.? 172.21.15.88:0/1934296928' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:29:48.717 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:48.717 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='client.? 172.21.15.88:0/2752310417' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:29:48.717 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:48.717 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:48.717 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='client.? 172.21.15.88:0/3752828360' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:29:48.717 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:29:48.717 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/trash_purge_schedule"}]: dispatch 2024-09-16T09:29:48.717 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:48.717 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:48.717 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='client.? 172.21.15.88:0/1952828841' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:29:48.718 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:48.718 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:48.718 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: Deploying daemon node-exporter.smithi078 on smithi078 2024-09-16T09:29:48.718 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='client.? 172.21.15.88:0/402764849' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:29:48.718 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:48.718 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:48.718 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:48.718 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-16T09:29:48.718 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14160 172.21.15.78:0/1780365802' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-16T09:29:48.718 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: mgrmap e14: smithi078.rrfxge(active, since 2m) 2024-09-16T09:29:48.718 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='client.? 172.21.15.88:0/2316460971' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:29:48.718 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='client.? 172.21.15.88:0/2320471094' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:29:48.718 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: Active manager daemon smithi078.rrfxge restarted 2024-09-16T09:29:48.718 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: Activating manager daemon smithi078.rrfxge 2024-09-16T09:29:48.718 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: osdmap e5: 0 total, 0 up, 0 in 2024-09-16T09:29:48.719 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: mgrmap e15: smithi078.rrfxge(active, starting, since 0.806282s) 2024-09-16T09:29:48.719 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-16T09:29:48.719 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mgr metadata", "who": "smithi078.rrfxge", "id": "smithi078.rrfxge"}]: dispatch 2024-09-16T09:29:48.719 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-16T09:29:48.719 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-16T09:29:48.719 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-16T09:29:48.719 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: Manager daemon smithi078.rrfxge is now available 2024-09-16T09:29:48.719 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:29:48.719 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:29:48.719 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:29:48.719 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:29:48.719 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:48.719 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:29:48.719 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/trash_purge_schedule"}]: dispatch 2024-09-16T09:29:48.720 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:48.720 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='client.? 172.21.15.88:0/4145372374' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:29:48.720 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: mgrmap e16: smithi078.rrfxge(active, since 1.80828s) 2024-09-16T09:29:48.720 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:48.720 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:48.720 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: [16/Sep/2024:09:29:30] ENGINE Bus STARTING 2024-09-16T09:29:48.720 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: [16/Sep/2024:09:29:30] ENGINE Serving on https://172.21.15.78:7150 2024-09-16T09:29:48.720 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: [16/Sep/2024:09:29:30] ENGINE Bus STARTED 2024-09-16T09:29:48.720 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: mgrmap e17: smithi078.rrfxge(active, since 2s) 2024-09-16T09:29:48.720 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:48.720 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:48.720 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:48.720 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd/host:smithi088", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:29:48.720 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:48.721 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='client.? 172.21.15.88:0/2954457276' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:29:48.721 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:48.721 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd/host:smithi078", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:29:48.721 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:48.721 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:48.721 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi088", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-16T09:29:48.721 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi088", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-16T09:29:48.721 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:29:48.721 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: Updating smithi078:/etc/ceph/ceph.conf 2024-09-16T09:29:48.721 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-09-16T09:29:48.721 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: Deploying daemon crash.smithi088 on smithi088 2024-09-16T09:29:48.721 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='client.? 172.21.15.88:0/1337184518' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:29:48.721 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:48.722 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi088.ncxfdz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T09:29:48.722 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi088.ncxfdz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-16T09:29:48.722 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-16T09:29:48.722 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:29:48.722 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: Deploying daemon mgr.smithi088.ncxfdz on smithi088 2024-09-16T09:29:48.722 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='client.? 172.21.15.88:0/2902094977' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:29:48.722 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:48.722 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-16T09:29:48.722 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:29:48.722 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: Deploying daemon mon.smithi088 on smithi088 2024-09-16T09:29:48.722 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:48.722 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 ceph-mon[29151]: mon.smithi088@-1(synchronizing).paxosservice(auth 1..6) refresh upgraded, format 0 -> 3 2024-09-16T09:29:48.722 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 bash[29094]: 774a14c001137d19729dd3c5b34889d2f9899d5b6fd9349c7cfefee18320bd7c 2024-09-16T09:29:48.722 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:48 smithi088 systemd[1]: Started Ceph mon.smithi088 for a70c765e-740d-11ef-bceb-c7b262605968. 2024-09-16T09:29:50.353 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T09:29:50.353 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T09:29:53.941 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:53 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-16T09:29:53.942 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:53 smithi088 ceph-mon[29151]: mon.smithi078 calling monitor election 2024-09-16T09:29:53.942 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:53 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi088"}]: dispatch 2024-09-16T09:29:53.942 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:53 smithi088 ceph-mon[29151]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:53.942 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:53 smithi088 ceph-mon[29151]: from='mgr.? 172.21.15.88:0/3793070365' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi088.ncxfdz/crt"}]: dispatch 2024-09-16T09:29:53.942 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:53 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi088"}]: dispatch 2024-09-16T09:29:53.942 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:53 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi088"}]: dispatch 2024-09-16T09:29:53.943 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:53 smithi088 ceph-mon[29151]: mon.smithi088 calling monitor election 2024-09-16T09:29:53.943 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:53 smithi088 ceph-mon[29151]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:53.943 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:53 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi088"}]: dispatch 2024-09-16T09:29:53.943 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:53 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi088"}]: dispatch 2024-09-16T09:29:53.943 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:53 smithi088 ceph-mon[29151]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:53.943 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:53 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi088"}]: dispatch 2024-09-16T09:29:53.944 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:53 smithi088 ceph-mon[29151]: mon.smithi078 is new leader, mons smithi078,smithi088 in quorum (ranks 0,1) 2024-09-16T09:29:53.944 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:53 smithi088 ceph-mon[29151]: monmap e2: 2 mons at {smithi078=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0],smithi088=[v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0]} 2024-09-16T09:29:53.944 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:53 smithi088 ceph-mon[29151]: fsmap 2024-09-16T09:29:53.944 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:53 smithi088 ceph-mon[29151]: osdmap e5: 0 total, 0 up, 0 in 2024-09-16T09:29:53.944 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:53 smithi088 ceph-mon[29151]: mgrmap e17: smithi078.rrfxge(active, since 25s) 2024-09-16T09:29:53.944 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:53 smithi088 ceph-mon[29151]: Standby manager daemon smithi088.ncxfdz started 2024-09-16T09:29:53.945 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:53 smithi088 ceph-mon[29151]: from='mgr.? 172.21.15.88:0/3793070365' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-16T09:29:53.945 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:53 smithi088 ceph-mon[29151]: overall HEALTH_OK 2024-09-16T09:29:53.945 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:53 smithi088 ceph-mon[29151]: from='mgr.? 172.21.15.88:0/3793070365' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi088.ncxfdz/key"}]: dispatch 2024-09-16T09:29:53.945 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:53 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:53.946 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:53 smithi088 ceph-mon[29151]: from='mgr.? 172.21.15.88:0/3793070365' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-16T09:29:54.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:53 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-16T09:29:54.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:53 smithi078 ceph-mon[23694]: mon.smithi078 calling monitor election 2024-09-16T09:29:54.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:53 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi088"}]: dispatch 2024-09-16T09:29:54.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:53 smithi078 ceph-mon[23694]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:54.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:53 smithi078 ceph-mon[23694]: from='mgr.? 172.21.15.88:0/3793070365' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi088.ncxfdz/crt"}]: dispatch 2024-09-16T09:29:54.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:53 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi088"}]: dispatch 2024-09-16T09:29:54.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:53 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi088"}]: dispatch 2024-09-16T09:29:54.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:53 smithi078 ceph-mon[23694]: mon.smithi088 calling monitor election 2024-09-16T09:29:54.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:53 smithi078 ceph-mon[23694]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:54.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:53 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi088"}]: dispatch 2024-09-16T09:29:54.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:53 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi088"}]: dispatch 2024-09-16T09:29:54.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:53 smithi078 ceph-mon[23694]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:54.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:53 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi088"}]: dispatch 2024-09-16T09:29:54.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:53 smithi078 ceph-mon[23694]: mon.smithi078 is new leader, mons smithi078,smithi088 in quorum (ranks 0,1) 2024-09-16T09:29:54.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:53 smithi078 ceph-mon[23694]: monmap e2: 2 mons at {smithi078=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0],smithi088=[v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0]} 2024-09-16T09:29:54.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:53 smithi078 ceph-mon[23694]: fsmap 2024-09-16T09:29:54.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:53 smithi078 ceph-mon[23694]: osdmap e5: 0 total, 0 up, 0 in 2024-09-16T09:29:54.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:53 smithi078 ceph-mon[23694]: mgrmap e17: smithi078.rrfxge(active, since 25s) 2024-09-16T09:29:54.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:53 smithi078 ceph-mon[23694]: Standby manager daemon smithi088.ncxfdz started 2024-09-16T09:29:54.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:53 smithi078 ceph-mon[23694]: from='mgr.? 172.21.15.88:0/3793070365' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-16T09:29:54.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:53 smithi078 ceph-mon[23694]: overall HEALTH_OK 2024-09-16T09:29:54.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:53 smithi078 ceph-mon[23694]: from='mgr.? 172.21.15.88:0/3793070365' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi088.ncxfdz/key"}]: dispatch 2024-09-16T09:29:54.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:53 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:54.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:53 smithi078 ceph-mon[23694]: from='mgr.? 172.21.15.88:0/3793070365' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-16T09:29:55.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:54 smithi088 ceph-mon[29151]: Deploying daemon node-exporter.smithi088 on smithi088 2024-09-16T09:29:55.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:54 smithi088 ceph-mon[29151]: mgrmap e18: smithi078.rrfxge(active, since 25s), standbys: smithi088.ncxfdz 2024-09-16T09:29:55.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:54 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mgr metadata", "who": "smithi088.ncxfdz", "id": "smithi088.ncxfdz"}]: dispatch 2024-09-16T09:29:55.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:54 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:55.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:54 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi088"}]: dispatch 2024-09-16T09:29:55.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:54 smithi078 ceph-mon[23694]: Deploying daemon node-exporter.smithi088 on smithi088 2024-09-16T09:29:55.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:54 smithi078 ceph-mon[23694]: mgrmap e18: smithi078.rrfxge(active, since 25s), standbys: smithi088.ncxfdz 2024-09-16T09:29:55.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:54 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mgr metadata", "who": "smithi088.ncxfdz", "id": "smithi088.ncxfdz"}]: dispatch 2024-09-16T09:29:55.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:54 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:55.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:54 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi088"}]: dispatch 2024-09-16T09:29:55.280 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:29:55.280 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":2,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","modified":"2024-09-16T09:29:48.639329Z","created":"2024-09-16T09:26:06.341058Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi088","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:3300","nonce":0},{"type":"v1","addr":"172.21.15.88:6789","nonce":0}]},"addr":"172.21.15.88:6789/0","public_addr":"172.21.15.88:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-16T09:29:55.281 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 2 2024-09-16T09:29:56.037 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:55 smithi088 ceph-mon[29151]: from='client.? 172.21.15.88:0/4281246376' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:29:56.037 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:55 smithi088 ceph-mon[29151]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:56.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:55 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/4281246376' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T09:29:56.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:55 smithi078 ceph-mon[23694]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:56.741 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-16T09:29:56.742 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph config generate-minimal-conf 2024-09-16T09:29:58.227 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:58 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:58.227 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:58 smithi078 ceph-mon[23694]: Deploying daemon prometheus.smithi078 on smithi078 2024-09-16T09:29:58.227 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:58 smithi078 ceph-mon[23694]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:58.550 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:58 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:29:58.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:58 smithi088 ceph-mon[29151]: Deploying daemon prometheus.smithi078 on smithi078 2024-09-16T09:29:58.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:58 smithi088 ceph-mon[29151]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:29:58.608 INFO:teuthology.orchestra.run.smithi078.stdout:# minimal ceph.conf for a70c765e-740d-11ef-bceb-c7b262605968 2024-09-16T09:29:58.608 INFO:teuthology.orchestra.run.smithi078.stdout:[global] 2024-09-16T09:29:58.609 INFO:teuthology.orchestra.run.smithi078.stdout: fsid = a70c765e-740d-11ef-bceb-c7b262605968 2024-09-16T09:29:58.609 INFO:teuthology.orchestra.run.smithi078.stdout: mon_host = [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] [v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0] 2024-09-16T09:29:59.580 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-16T09:29:59.581 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-16T09:29:59.581 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/ceph/ceph.conf 2024-09-16T09:29:59.587 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:29:59 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/405163812' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:29:59.612 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-16T09:29:59.612 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-16T09:29:59.683 DEBUG:teuthology.orchestra.run.smithi088:> set -ex 2024-09-16T09:29:59.683 DEBUG:teuthology.orchestra.run.smithi088:> sudo dd of=/etc/ceph/ceph.conf 2024-09-16T09:29:59.712 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:29:59 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/405163812' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:29:59.717 DEBUG:teuthology.orchestra.run.smithi088:> set -ex 2024-09-16T09:29:59.717 DEBUG:teuthology.orchestra.run.smithi088:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-16T09:29:59.785 DEBUG:tasks.cephadm:set 0 configs 2024-09-16T09:29:59.785 INFO:tasks.cephadm:Deploying OSDs... 2024-09-16T09:29:59.786 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-16T09:29:59.786 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/scratch_devs of=/dev/stdout 2024-09-16T09:29:59.804 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-16T09:29:59.804 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_1 2024-09-16T09:29:59.861 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-16T09:29:59.861 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-16T09:29:59.861 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-09-16T09:29:59.861 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-16T09:29:59.861 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-09-16T09:29:59.861 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-09-16 09:29:19.138284625 +0000 2024-09-16T09:29:59.861 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-09-16 09:25:25.012032817 +0000 2024-09-16T09:29:59.862 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-09-16 09:25:25.012032817 +0000 2024-09-16T09:29:59.862 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-09-16 09:25:25.012032817 +0000 2024-09-16T09:29:59.862 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-16T09:29:59.925 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-09-16T09:29:59.926 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-09-16T09:29:59.926 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000217239 s, 2.4 MB/s 2024-09-16T09:29:59.927 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-16T09:29:59.983 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_2 2024-09-16T09:30:00.040 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-16T09:30:00.041 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-16T09:30:00.041 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-09-16T09:30:00.041 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-16T09:30:00.041 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-09-16T09:30:00.041 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-09-16 09:29:19.138284625 +0000 2024-09-16T09:30:00.042 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-09-16 09:25:25.364027324 +0000 2024-09-16T09:30:00.042 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-09-16 09:25:25.364027324 +0000 2024-09-16T09:30:00.042 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-09-16 09:25:25.364027324 +0000 2024-09-16T09:30:00.042 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-16T09:30:00.105 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-09-16T09:30:00.105 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-09-16T09:30:00.105 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000223439 s, 2.3 MB/s 2024-09-16T09:30:00.107 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-16T09:30:00.163 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_3 2024-09-16T09:30:00.219 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-16T09:30:00.219 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-16T09:30:00.219 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 704 Links: 1 2024-09-16T09:30:00.220 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-16T09:30:00.220 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-09-16T09:30:00.220 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-09-16 09:29:19.139284609 +0000 2024-09-16T09:30:00.221 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-09-16 09:25:25.693022190 +0000 2024-09-16T09:30:00.221 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-09-16 09:25:25.693022190 +0000 2024-09-16T09:30:00.221 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-09-16 09:25:25.693022190 +0000 2024-09-16T09:30:00.222 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-16T09:30:00.284 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-09-16T09:30:00.285 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-09-16T09:30:00.285 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000169131 s, 3.0 MB/s 2024-09-16T09:30:00.286 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-16T09:30:00.343 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_4 2024-09-16T09:30:00.400 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-16T09:30:00.400 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-16T09:30:00.400 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-09-16T09:30:00.400 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-16T09:30:00.400 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-09-16T09:30:00.400 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-09-16 09:29:19.139284609 +0000 2024-09-16T09:30:00.400 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-09-16 09:25:26.021017072 +0000 2024-09-16T09:30:00.400 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-09-16 09:25:26.021017072 +0000 2024-09-16T09:30:00.400 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-09-16 09:25:26.021017072 +0000 2024-09-16T09:30:00.401 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-16T09:30:00.469 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-09-16T09:30:00.469 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-09-16T09:30:00.469 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000212768 s, 2.4 MB/s 2024-09-16T09:30:00.470 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-16T09:30:00.531 DEBUG:teuthology.orchestra.run.smithi088:> set -ex 2024-09-16T09:30:00.531 DEBUG:teuthology.orchestra.run.smithi088:> dd if=/scratch_devs of=/dev/stdout 2024-09-16T09:30:00.551 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-16T09:30:00.551 DEBUG:teuthology.orchestra.run.smithi088:> stat /dev/vg_nvme/lv_1 2024-09-16T09:30:00.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:00 smithi078 ceph-mon[23694]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:00.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:00 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:00.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:00 smithi078 ceph-mon[23694]: overall HEALTH_OK 2024-09-16T09:30:00.609 INFO:teuthology.orchestra.run.smithi088.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-16T09:30:00.609 INFO:teuthology.orchestra.run.smithi088.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-16T09:30:00.609 INFO:teuthology.orchestra.run.smithi088.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-09-16T09:30:00.609 INFO:teuthology.orchestra.run.smithi088.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-16T09:30:00.609 INFO:teuthology.orchestra.run.smithi088.stdout:Context: system_u:object_r:device_t:s0 2024-09-16T09:30:00.610 INFO:teuthology.orchestra.run.smithi088.stdout:Access: 2024-09-16 09:29:54.231844226 +0000 2024-09-16T09:30:00.610 INFO:teuthology.orchestra.run.smithi088.stdout:Modify: 2024-09-16 09:25:25.006070577 +0000 2024-09-16T09:30:00.610 INFO:teuthology.orchestra.run.smithi088.stdout:Change: 2024-09-16 09:25:25.006070577 +0000 2024-09-16T09:30:00.610 INFO:teuthology.orchestra.run.smithi088.stdout: Birth: 2024-09-16 09:25:25.006070577 +0000 2024-09-16T09:30:00.610 DEBUG:teuthology.orchestra.run.smithi088:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-16T09:30:00.673 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:00 smithi088 ceph-mon[29151]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:00.673 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:00 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:00.673 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:00 smithi088 ceph-mon[29151]: overall HEALTH_OK 2024-09-16T09:30:00.676 INFO:teuthology.orchestra.run.smithi088.stderr:1+0 records in 2024-09-16T09:30:00.676 INFO:teuthology.orchestra.run.smithi088.stderr:1+0 records out 2024-09-16T09:30:00.676 INFO:teuthology.orchestra.run.smithi088.stderr:512 bytes copied, 0.000220792 s, 2.3 MB/s 2024-09-16T09:30:00.678 DEBUG:teuthology.orchestra.run.smithi088:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-16T09:30:00.737 DEBUG:teuthology.orchestra.run.smithi088:> stat /dev/vg_nvme/lv_2 2024-09-16T09:30:00.795 INFO:teuthology.orchestra.run.smithi088.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-16T09:30:00.795 INFO:teuthology.orchestra.run.smithi088.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-16T09:30:00.795 INFO:teuthology.orchestra.run.smithi088.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-09-16T09:30:00.795 INFO:teuthology.orchestra.run.smithi088.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-16T09:30:00.795 INFO:teuthology.orchestra.run.smithi088.stdout:Context: system_u:object_r:device_t:s0 2024-09-16T09:30:00.795 INFO:teuthology.orchestra.run.smithi088.stdout:Access: 2024-09-16 09:29:54.231844226 +0000 2024-09-16T09:30:00.795 INFO:teuthology.orchestra.run.smithi088.stdout:Modify: 2024-09-16 09:25:25.346064218 +0000 2024-09-16T09:30:00.795 INFO:teuthology.orchestra.run.smithi088.stdout:Change: 2024-09-16 09:25:25.346064218 +0000 2024-09-16T09:30:00.796 INFO:teuthology.orchestra.run.smithi088.stdout: Birth: 2024-09-16 09:25:25.346064218 +0000 2024-09-16T09:30:00.796 DEBUG:teuthology.orchestra.run.smithi088:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-16T09:30:00.861 INFO:teuthology.orchestra.run.smithi088.stderr:1+0 records in 2024-09-16T09:30:00.861 INFO:teuthology.orchestra.run.smithi088.stderr:1+0 records out 2024-09-16T09:30:00.861 INFO:teuthology.orchestra.run.smithi088.stderr:512 bytes copied, 0.000201278 s, 2.5 MB/s 2024-09-16T09:30:00.862 DEBUG:teuthology.orchestra.run.smithi088:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-16T09:30:00.920 DEBUG:teuthology.orchestra.run.smithi088:> stat /dev/vg_nvme/lv_3 2024-09-16T09:30:00.976 INFO:teuthology.orchestra.run.smithi088.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-16T09:30:00.976 INFO:teuthology.orchestra.run.smithi088.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-16T09:30:00.976 INFO:teuthology.orchestra.run.smithi088.stdout:Device: 5h/5d Inode: 704 Links: 1 2024-09-16T09:30:00.977 INFO:teuthology.orchestra.run.smithi088.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-16T09:30:00.977 INFO:teuthology.orchestra.run.smithi088.stdout:Context: system_u:object_r:device_t:s0 2024-09-16T09:30:00.977 INFO:teuthology.orchestra.run.smithi088.stdout:Access: 2024-09-16 09:29:54.231844226 +0000 2024-09-16T09:30:00.977 INFO:teuthology.orchestra.run.smithi088.stdout:Modify: 2024-09-16 09:25:25.688057821 +0000 2024-09-16T09:30:00.977 INFO:teuthology.orchestra.run.smithi088.stdout:Change: 2024-09-16 09:25:25.688057821 +0000 2024-09-16T09:30:00.977 INFO:teuthology.orchestra.run.smithi088.stdout: Birth: 2024-09-16 09:25:25.688057821 +0000 2024-09-16T09:30:00.977 DEBUG:teuthology.orchestra.run.smithi088:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-16T09:30:01.041 INFO:teuthology.orchestra.run.smithi088.stderr:1+0 records in 2024-09-16T09:30:01.042 INFO:teuthology.orchestra.run.smithi088.stderr:1+0 records out 2024-09-16T09:30:01.043 INFO:teuthology.orchestra.run.smithi088.stderr:512 bytes copied, 0.000204756 s, 2.5 MB/s 2024-09-16T09:30:01.044 DEBUG:teuthology.orchestra.run.smithi088:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-16T09:30:01.103 DEBUG:teuthology.orchestra.run.smithi088:> stat /dev/vg_nvme/lv_4 2024-09-16T09:30:01.159 INFO:teuthology.orchestra.run.smithi088.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-16T09:30:01.159 INFO:teuthology.orchestra.run.smithi088.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-16T09:30:01.159 INFO:teuthology.orchestra.run.smithi088.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-09-16T09:30:01.159 INFO:teuthology.orchestra.run.smithi088.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-16T09:30:01.159 INFO:teuthology.orchestra.run.smithi088.stdout:Context: system_u:object_r:device_t:s0 2024-09-16T09:30:01.159 INFO:teuthology.orchestra.run.smithi088.stdout:Access: 2024-09-16 09:29:54.232844206 +0000 2024-09-16T09:30:01.159 INFO:teuthology.orchestra.run.smithi088.stdout:Modify: 2024-09-16 09:25:26.024051537 +0000 2024-09-16T09:30:01.159 INFO:teuthology.orchestra.run.smithi088.stdout:Change: 2024-09-16 09:25:26.024051537 +0000 2024-09-16T09:30:01.159 INFO:teuthology.orchestra.run.smithi088.stdout: Birth: 2024-09-16 09:25:26.024051537 +0000 2024-09-16T09:30:01.160 DEBUG:teuthology.orchestra.run.smithi088:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-16T09:30:01.222 INFO:teuthology.orchestra.run.smithi088.stderr:1+0 records in 2024-09-16T09:30:01.223 INFO:teuthology.orchestra.run.smithi088.stderr:1+0 records out 2024-09-16T09:30:01.223 INFO:teuthology.orchestra.run.smithi088.stderr:512 bytes copied, 0.000201903 s, 2.5 MB/s 2024-09-16T09:30:01.224 DEBUG:teuthology.orchestra.run.smithi088:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-16T09:30:01.284 INFO:tasks.cephadm:Deploying osd.0 on smithi078 with /dev/vg_nvme/lv_4... 2024-09-16T09:30:01.284 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-16T09:30:02.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:02 smithi078 ceph-mon[23694]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:02.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:02 smithi088 ceph-mon[29151]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:04.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:04 smithi088 ceph-mon[29151]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:04.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:04 smithi078 ceph-mon[23694]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:06.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:06 smithi088 ceph-mon[29151]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:06.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:06 smithi078 ceph-mon[23694]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:08.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:08 smithi078 ceph-mon[23694]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:08.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:08 smithi088 ceph-mon[29151]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:10.777 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:10 smithi078 ceph-mon[23694]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:10.777 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:10 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:10.777 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:10 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:30:10.777 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:10 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:30:10.778 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:10 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:30:10.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:10 smithi088 ceph-mon[29151]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:10.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:10 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:10.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:10 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:30:10.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:10 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:30:10.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:10 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:30:11.537 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-16T09:30:11.559 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi078:vg_nvme/lv_4 2024-09-16T09:30:12.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:12 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:12.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:12 smithi078 ceph-mon[23694]: Updating smithi088:/etc/ceph/ceph.conf 2024-09-16T09:30:12.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:12 smithi078 ceph-mon[23694]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:12.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:12 smithi078 ceph-mon[23694]: Updating smithi088:/etc/ceph/ceph.client.admin.keyring 2024-09-16T09:30:12.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:12 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:12.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:12 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:12.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:12 smithi088 ceph-mon[29151]: Updating smithi088:/etc/ceph/ceph.conf 2024-09-16T09:30:12.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:12 smithi088 ceph-mon[29151]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:12.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:12 smithi088 ceph-mon[29151]: Updating smithi088:/etc/ceph/ceph.client.admin.keyring 2024-09-16T09:30:12.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:12 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:14.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:14 smithi078 ceph-mon[23694]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:14.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:14 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T09:30:14.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:14 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T09:30:14.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:14 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:30:14.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:14 smithi088 ceph-mon[29151]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:14.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:14 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T09:30:14.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:14 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T09:30:14.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:14 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:30:15.621 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:15 smithi078 ceph-mon[23694]: from='client.14286 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:30:15.622 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:15 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:15.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:15 smithi088 ceph-mon[29151]: from='client.14286 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:30:15.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:15 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:16.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:16 smithi078 ceph-mon[23694]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:16.683 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:16 smithi088 ceph-mon[29151]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:17.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:17 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/1159174128' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9c561ce9-358c-4e1f-aaa0-497e8410c604"}]: dispatch 2024-09-16T09:30:17.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:17 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/1159174128' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9c561ce9-358c-4e1f-aaa0-497e8410c604"}]': finished 2024-09-16T09:30:17.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:17 smithi088 ceph-mon[29151]: osdmap e6: 1 total, 0 up, 1 in 2024-09-16T09:30:17.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:17 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T09:30:17.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:17 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:17.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:17 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:17.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:17 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:17.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:17 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-16T09:30:17.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:17 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-16T09:30:17.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:17 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:30:17.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:17 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/1159174128' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9c561ce9-358c-4e1f-aaa0-497e8410c604"}]: dispatch 2024-09-16T09:30:17.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:17 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/1159174128' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9c561ce9-358c-4e1f-aaa0-497e8410c604"}]': finished 2024-09-16T09:30:17.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:17 smithi078 ceph-mon[23694]: osdmap e6: 1 total, 0 up, 1 in 2024-09-16T09:30:17.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:17 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T09:30:17.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:17 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:17.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:17 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:17.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:17 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:17.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:17 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-16T09:30:17.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:17 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-16T09:30:17.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:17 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:30:18.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:18 smithi088 ceph-mon[29151]: Updating smithi078:/etc/ceph/ceph.conf 2024-09-16T09:30:18.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:18 smithi088 ceph-mon[29151]: Reconfiguring mon.smithi078 (unknown last config time)... 2024-09-16T09:30:18.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:18 smithi088 ceph-mon[29151]: Reconfiguring daemon mon.smithi078 on smithi078 2024-09-16T09:30:18.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:18 smithi088 ceph-mon[29151]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:18.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:18 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/4029183320' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T09:30:18.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:18 smithi078 ceph-mon[23694]: Updating smithi078:/etc/ceph/ceph.conf 2024-09-16T09:30:18.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:18 smithi078 ceph-mon[23694]: Reconfiguring mon.smithi078 (unknown last config time)... 2024-09-16T09:30:18.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:18 smithi078 ceph-mon[23694]: Reconfiguring daemon mon.smithi078 on smithi078 2024-09-16T09:30:18.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:18 smithi078 ceph-mon[23694]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:18.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:18 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/4029183320' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T09:30:20.491 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:20 smithi078 ceph-mon[23694]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:20.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:20 smithi088 ceph-mon[29151]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:22.638 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:22 smithi078 ceph-mon[23694]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:22.638 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:22 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:22.638 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:22 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.rrfxge", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T09:30:22.638 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:22 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-16T09:30:22.638 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:22 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:30:22.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:22 smithi088 ceph-mon[29151]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:22.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:22 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:22.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:22 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.rrfxge", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T09:30:22.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:22 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-16T09:30:22.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:22 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:30:23.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:23 smithi088 ceph-mon[29151]: Reconfiguring mgr.smithi078.rrfxge (unknown last config time)... 2024-09-16T09:30:23.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:23 smithi088 ceph-mon[29151]: Reconfiguring daemon mgr.smithi078.rrfxge on smithi078 2024-09-16T09:30:23.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:23 smithi078 ceph-mon[23694]: Reconfiguring mgr.smithi078.rrfxge (unknown last config time)... 2024-09-16T09:30:23.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:23 smithi078 ceph-mon[23694]: Reconfiguring daemon mgr.smithi078.rrfxge on smithi078 2024-09-16T09:30:24.769 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:24 smithi088 ceph-mon[29151]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:24.769 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:24.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:24 smithi078 ceph-mon[23694]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:24.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:25.711 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:25 smithi078 ceph-mon[23694]: Reconfiguring alertmanager.smithi078 (dependencies changed)... 2024-09-16T09:30:25.711 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:25 smithi078 ceph-mon[23694]: Reconfiguring daemon alertmanager.smithi078 on smithi078 2024-09-16T09:30:25.711 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:25 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-16T09:30:25.711 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:25 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:30:25.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:25 smithi088 ceph-mon[29151]: Reconfiguring alertmanager.smithi078 (dependencies changed)... 2024-09-16T09:30:25.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:25 smithi088 ceph-mon[29151]: Reconfiguring daemon alertmanager.smithi078 on smithi078 2024-09-16T09:30:25.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:25 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-16T09:30:25.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:25 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:30:26.686 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:26 smithi078 ceph-mon[23694]: Deploying daemon osd.0 on smithi078 2024-09-16T09:30:26.686 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:26 smithi078 ceph-mon[23694]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:26.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:26 smithi088 ceph-mon[29151]: Deploying daemon osd.0 on smithi078 2024-09-16T09:30:26.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:26 smithi088 ceph-mon[29151]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:28.674 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:28 smithi078 ceph-mon[23694]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:28.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:28 smithi088 ceph-mon[29151]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:29.601 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:29 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:30:29.601 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:29 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/trash_purge_schedule"}]: dispatch 2024-09-16T09:30:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:29 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:30:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:29 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/trash_purge_schedule"}]: dispatch 2024-09-16T09:30:30.553 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:30 smithi078 ceph-mon[23694]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:30.554 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:30 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:30.554 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:30 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-16T09:30:30.554 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:30 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:30:30.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:30 smithi088 ceph-mon[29151]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:30.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:30 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:30.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:30 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-16T09:30:30.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:30 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:30:31.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:31 smithi088 ceph-mon[29151]: Reconfiguring crash.smithi078 (monmap changed)... 2024-09-16T09:30:31.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:31 smithi088 ceph-mon[29151]: Reconfiguring daemon crash.smithi078 on smithi078 2024-09-16T09:30:31.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:31 smithi078 ceph-mon[23694]: Reconfiguring crash.smithi078 (monmap changed)... 2024-09-16T09:30:31.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:31 smithi078 ceph-mon[23694]: Reconfiguring daemon crash.smithi078 on smithi078 2024-09-16T09:30:32.434 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:32 smithi078 ceph-mon[23694]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:32.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:32 smithi088 ceph-mon[29151]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:34.550 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:34 smithi088 ceph-mon[29151]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:34.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:34 smithi078 ceph-mon[23694]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:35.933 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:35 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:35.934 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:35 smithi078 ceph-mon[23694]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:36.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:35 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:36.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:35 smithi088 ceph-mon[29151]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:37.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:36 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:37.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:36 smithi088 ceph-mon[29151]: Reconfiguring grafana.smithi078 (dependencies changed)... 2024-09-16T09:30:37.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:36 smithi088 ceph-mon[29151]: Reconfiguring daemon grafana.smithi078 on smithi078 2024-09-16T09:30:37.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:36 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:37.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:36 smithi078 ceph-mon[23694]: Reconfiguring grafana.smithi078 (dependencies changed)... 2024-09-16T09:30:37.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:36 smithi078 ceph-mon[23694]: Reconfiguring daemon grafana.smithi078 on smithi078 2024-09-16T09:30:38.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:37 smithi088 ceph-mon[29151]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:38.055 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:37 smithi078 ceph-mon[23694]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:39.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:38 smithi088 ceph-mon[29151]: from='osd.0 [v2:172.21.15.78:6802/3273201876,v1:172.21.15.78:6803/3273201876]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-16T09:30:39.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:38 smithi078 ceph-mon[23694]: from='osd.0 [v2:172.21.15.78:6802/3273201876,v1:172.21.15.78:6803/3273201876]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-16T09:30:39.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:39 smithi078 ceph-mon[23694]: from='osd.0 [v2:172.21.15.78:6802/3273201876,v1:172.21.15.78:6803/3273201876]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-16T09:30:39.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:39 smithi078 ceph-mon[23694]: osdmap e7: 1 total, 0 up, 1 in 2024-09-16T09:30:39.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:39 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T09:30:39.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:39 smithi078 ceph-mon[23694]: from='osd.0 [v2:172.21.15.78:6802/3273201876,v1:172.21.15.78:6803/3273201876]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-09-16T09:30:39.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:39 smithi078 ceph-mon[23694]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:40.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:39 smithi088 ceph-mon[29151]: from='osd.0 [v2:172.21.15.78:6802/3273201876,v1:172.21.15.78:6803/3273201876]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-16T09:30:40.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:39 smithi088 ceph-mon[29151]: osdmap e7: 1 total, 0 up, 1 in 2024-09-16T09:30:40.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:39 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T09:30:40.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:39 smithi088 ceph-mon[29151]: from='osd.0 [v2:172.21.15.78:6802/3273201876,v1:172.21.15.78:6803/3273201876]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-09-16T09:30:40.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:39 smithi088 ceph-mon[29151]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:40.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:40 smithi088 ceph-mon[29151]: from='osd.0 [v2:172.21.15.78:6802/3273201876,v1:172.21.15.78:6803/3273201876]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-09-16T09:30:40.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:40 smithi088 ceph-mon[29151]: osdmap e8: 1 total, 0 up, 1 in 2024-09-16T09:30:40.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:40 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T09:30:40.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:40 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T09:30:41.023 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:40 smithi078 ceph-mon[23694]: from='osd.0 [v2:172.21.15.78:6802/3273201876,v1:172.21.15.78:6803/3273201876]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-09-16T09:30:41.023 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:40 smithi078 ceph-mon[23694]: osdmap e8: 1 total, 0 up, 1 in 2024-09-16T09:30:41.024 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:40 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T09:30:41.024 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:40 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T09:30:41.803 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:41 smithi078 ceph-mon[23694]: purged_snaps scrub starts 2024-09-16T09:30:41.803 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:41 smithi078 ceph-mon[23694]: purged_snaps scrub ok 2024-09-16T09:30:41.803 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:41 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T09:30:41.803 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:41 smithi078 ceph-mon[23694]: osd.0 [v2:172.21.15.78:6802/3273201876,v1:172.21.15.78:6803/3273201876] boot 2024-09-16T09:30:41.803 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:41 smithi078 ceph-mon[23694]: osdmap e9: 1 total, 1 up, 1 in 2024-09-16T09:30:41.804 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:41 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T09:30:41.804 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:41 smithi078 ceph-mon[23694]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:41.804 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:41 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:41.804 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:41 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi088", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-16T09:30:41.804 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:41 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:30:41.980 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:41 smithi088 ceph-mon[29151]: purged_snaps scrub starts 2024-09-16T09:30:41.980 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:41 smithi088 ceph-mon[29151]: purged_snaps scrub ok 2024-09-16T09:30:41.980 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:41 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T09:30:41.980 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:41 smithi088 ceph-mon[29151]: osd.0 [v2:172.21.15.78:6802/3273201876,v1:172.21.15.78:6803/3273201876] boot 2024-09-16T09:30:41.981 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:41 smithi088 ceph-mon[29151]: osdmap e9: 1 total, 1 up, 1 in 2024-09-16T09:30:41.981 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:41 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T09:30:41.981 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:41 smithi088 ceph-mon[29151]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T09:30:41.981 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:41 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:41.981 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:41 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi088", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-16T09:30:41.981 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:41 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:30:42.801 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:42 smithi078 ceph-mon[23694]: Reconfiguring crash.smithi088 (monmap changed)... 2024-09-16T09:30:42.801 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:42 smithi078 ceph-mon[23694]: Reconfiguring daemon crash.smithi088 on smithi088 2024-09-16T09:30:42.802 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:42 smithi078 ceph-mon[23694]: osdmap e10: 1 total, 1 up, 1 in 2024-09-16T09:30:42.956 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:42 smithi088 ceph-mon[29151]: Reconfiguring crash.smithi088 (monmap changed)... 2024-09-16T09:30:42.957 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:42 smithi088 ceph-mon[29151]: Reconfiguring daemon crash.smithi088 on smithi088 2024-09-16T09:30:42.957 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:42 smithi088 ceph-mon[29151]: osdmap e10: 1 total, 1 up, 1 in 2024-09-16T09:30:43.098 INFO:teuthology.orchestra.run.smithi078.stdout:Created osd(s) 0 on host 'smithi078' 2024-09-16T09:30:43.745 DEBUG:teuthology.orchestra.run.smithi078:osd.0> sudo journalctl -f -n 0 -u ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.0.service 2024-09-16T09:30:43.748 INFO:tasks.cephadm:Deploying osd.1 on smithi078 with /dev/vg_nvme/lv_3... 2024-09-16T09:30:43.749 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-16T09:30:44.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:43 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:44.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:43 smithi078 ceph-mon[23694]: Reconfiguring mgr.smithi088.ncxfdz (monmap changed)... 2024-09-16T09:30:44.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:43 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi088.ncxfdz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T09:30:44.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:43 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-16T09:30:44.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:43 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:30:44.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:43 smithi078 ceph-mon[23694]: Reconfiguring daemon mgr.smithi088.ncxfdz on smithi088 2024-09-16T09:30:44.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:43 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:44.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:43 smithi078 ceph-mon[23694]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:30:44.210 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:43 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:44.210 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:43 smithi088 ceph-mon[29151]: Reconfiguring mgr.smithi088.ncxfdz (monmap changed)... 2024-09-16T09:30:44.211 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:43 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi088.ncxfdz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T09:30:44.211 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:43 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-16T09:30:44.211 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:43 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:30:44.211 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:43 smithi088 ceph-mon[29151]: Reconfiguring daemon mgr.smithi088.ncxfdz on smithi088 2024-09-16T09:30:44.211 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:43 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:44.211 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:43 smithi088 ceph-mon[29151]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:30:45.339 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:45 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:45.339 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:45 smithi078 ceph-mon[23694]: Reconfiguring mon.smithi088 (monmap changed)... 2024-09-16T09:30:45.339 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:45 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-16T09:30:45.340 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:45 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-16T09:30:45.340 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:45 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:30:45.340 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:45 smithi078 ceph-mon[23694]: Reconfiguring daemon mon.smithi088 on smithi088 2024-09-16T09:30:45.471 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:45 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:45.472 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:45 smithi088 ceph-mon[29151]: Reconfiguring mon.smithi088 (monmap changed)... 2024-09-16T09:30:45.472 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:45 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-16T09:30:45.472 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:45 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-16T09:30:45.472 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:45 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:30:45.472 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:45 smithi088 ceph-mon[29151]: Reconfiguring daemon mon.smithi088 on smithi088 2024-09-16T09:30:46.441 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-16T09:30:46.460 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi078:vg_nvme/lv_3 2024-09-16T09:30:46.741 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:46 smithi078 ceph-mon[23694]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:30:46.741 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:46 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:46.741 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:46 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-16T09:30:46.741 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:46 smithi078 ceph-mon[23694]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-16T09:30:46.741 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:46 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.78:9093"}]: dispatch 2024-09-16T09:30:46.741 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:46 smithi078 ceph-mon[23694]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.78:9093"}]: dispatch 2024-09-16T09:30:46.741 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:46 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:46.742 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:46 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-16T09:30:46.742 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:46 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.78:3000"}]: dispatch 2024-09-16T09:30:46.742 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:46 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:46.742 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:46 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-16T09:30:46.742 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:46 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.78:9095"}]: dispatch 2024-09-16T09:30:46.742 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:46 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:46.742 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:46 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:30:46.742 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:46 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:30:46.742 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:46 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:30:46.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:46 smithi088 ceph-mon[29151]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:30:46.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:46 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:46.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:46 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-16T09:30:46.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:46 smithi088 ceph-mon[29151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-16T09:30:46.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:46 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.78:9093"}]: dispatch 2024-09-16T09:30:46.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:46 smithi088 ceph-mon[29151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.78:9093"}]: dispatch 2024-09-16T09:30:46.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:46 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:46.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:46 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-16T09:30:46.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:46 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.78:3000"}]: dispatch 2024-09-16T09:30:46.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:46 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:46.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:46 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-16T09:30:46.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:46 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.78:9095"}]: dispatch 2024-09-16T09:30:46.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:46 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:46.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:46 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:30:46.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:46 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:30:46.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:46 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:30:47.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:47 smithi088 ceph-mon[29151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-16T09:30:47.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:47 smithi088 ceph-mon[29151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.78:3000"}]: dispatch 2024-09-16T09:30:47.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:47 smithi088 ceph-mon[29151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-16T09:30:47.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:47 smithi088 ceph-mon[29151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.78:9095"}]: dispatch 2024-09-16T09:30:47.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:47 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:47.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:47 smithi078 ceph-mon[23694]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-16T09:30:47.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:47 smithi078 ceph-mon[23694]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.78:3000"}]: dispatch 2024-09-16T09:30:47.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:47 smithi078 ceph-mon[23694]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-16T09:30:47.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:47 smithi078 ceph-mon[23694]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.78:9095"}]: dispatch 2024-09-16T09:30:47.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:47 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:48.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:48 smithi088 ceph-mon[29151]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:30:48.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:48 smithi078 ceph-mon[23694]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:30:49.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:49 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T09:30:49.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:49 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T09:30:49.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:49 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:30:49.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:49 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T09:30:49.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:49 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T09:30:49.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:49 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:30:50.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:50 smithi088 ceph-mon[29151]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:30:50.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:50 smithi088 ceph-mon[29151]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:30:50.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:50 smithi078 ceph-mon[23694]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:30:50.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:50 smithi078 ceph-mon[23694]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:30:51.883 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:51 smithi078 ceph-mon[23694]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:30:52.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:51 smithi088 ceph-mon[29151]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:30:53.166 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:52 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/1655808297' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0af2d2ef-0835-4383-add0-fc6a751b627f"}]: dispatch 2024-09-16T09:30:53.167 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:52 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/1655808297' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0af2d2ef-0835-4383-add0-fc6a751b627f"}]': finished 2024-09-16T09:30:53.167 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:52 smithi078 ceph-mon[23694]: osdmap e11: 2 total, 1 up, 2 in 2024-09-16T09:30:53.167 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:52 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T09:30:53.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:52 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/1655808297' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0af2d2ef-0835-4383-add0-fc6a751b627f"}]: dispatch 2024-09-16T09:30:53.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:52 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/1655808297' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0af2d2ef-0835-4383-add0-fc6a751b627f"}]': finished 2024-09-16T09:30:53.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:52 smithi088 ceph-mon[29151]: osdmap e11: 2 total, 1 up, 2 in 2024-09-16T09:30:53.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:52 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T09:30:54.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:54 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:54.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:54 smithi088 ceph-mon[29151]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:30:54.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:54 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/3214975567' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T09:30:54.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:54 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:54.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:54 smithi078 ceph-mon[23694]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:30:54.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:54 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/3214975567' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T09:30:56.664 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:56 smithi078 ceph-mon[23694]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:30:56.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:56 smithi088 ceph-mon[29151]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:30:58.754 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:58 smithi078 ceph-mon[23694]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:30:58.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:58 smithi088 ceph-mon[29151]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:30:59.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:59 smithi078 ceph-mon[23694]: Detected new or changed devices on smithi078 2024-09-16T09:30:59.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:59 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:59.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:59 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:30:59.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:59 smithi078 ceph-mon[23694]: Adjusting osd_memory_target on smithi078 to 12864M 2024-09-16T09:30:59.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:59 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:59.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:59 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:30:59.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:59 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:30:59.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:59 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:30:59.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:59 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:30:59.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:30:59 smithi078 ceph-mon[23694]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:31:00.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:59 smithi088 ceph-mon[29151]: Detected new or changed devices on smithi078 2024-09-16T09:31:00.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:59 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:00.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:59 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:31:00.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:59 smithi088 ceph-mon[29151]: Adjusting osd_memory_target on smithi078 to 12864M 2024-09-16T09:31:00.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:59 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:00.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:59 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:00.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:59 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:31:00.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:59 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:31:00.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:59 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:31:00.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:30:59 smithi088 ceph-mon[29151]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:31:01.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:00 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-16T09:31:01.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:00 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:31:01.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:00 smithi078 ceph-mon[23694]: Deploying daemon osd.1 on smithi078 2024-09-16T09:31:01.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:00 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:01.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:00 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-16T09:31:01.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:00 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:31:01.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:00 smithi088 ceph-mon[29151]: Deploying daemon osd.1 on smithi078 2024-09-16T09:31:01.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:00 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:01.931 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:01 smithi078 ceph-mon[23694]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:31:02.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:01 smithi088 ceph-mon[29151]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:31:04.637 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:04 smithi088 ceph-mon[29151]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:31:04.658 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:04 smithi078 ceph-mon[23694]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:31:06.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:06 smithi078 ceph-mon[23694]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:31:06.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:06 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:06.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:06 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:06.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:06 smithi088 ceph-mon[29151]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:31:06.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:06 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:06.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:06 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:08.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:08 smithi078 ceph-mon[23694]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:31:08.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:08 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:08.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:08 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:31:08.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:08 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:31:08.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:08 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:31:08.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:08 smithi088 ceph-mon[29151]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:31:08.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:08 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:08.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:08 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:31:08.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:08 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:31:08.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:08 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:31:10.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:10 smithi078 ceph-mon[23694]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:31:10.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:10 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:10.693 INFO:teuthology.orchestra.run.smithi078.stdout:Created osd(s) 1 on host 'smithi078' 2024-09-16T09:31:10.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:10 smithi088 ceph-mon[29151]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:31:10.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:10 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:11.258 DEBUG:teuthology.orchestra.run.smithi078:osd.1> sudo journalctl -f -n 0 -u ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.1.service 2024-09-16T09:31:11.261 INFO:tasks.cephadm:Deploying osd.2 on smithi078 with /dev/vg_nvme/lv_2... 2024-09-16T09:31:11.261 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-16T09:31:12.014 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:11 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:12.014 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:11 smithi078 ceph-mon[23694]: from='osd.1 [v2:172.21.15.78:6810/3035110387,v1:172.21.15.78:6811/3035110387]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-16T09:31:12.015 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:11 smithi078 ceph-mon[23694]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:31:12.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:11 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:12.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:11 smithi088 ceph-mon[29151]: from='osd.1 [v2:172.21.15.78:6810/3035110387,v1:172.21.15.78:6811/3035110387]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-16T09:31:12.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:11 smithi088 ceph-mon[29151]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:31:12.962 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:12 smithi078 ceph-mon[23694]: from='osd.1 [v2:172.21.15.78:6810/3035110387,v1:172.21.15.78:6811/3035110387]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-16T09:31:12.963 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:12 smithi078 ceph-mon[23694]: osdmap e12: 2 total, 1 up, 2 in 2024-09-16T09:31:12.963 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:12 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T09:31:12.963 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:12 smithi078 ceph-mon[23694]: from='osd.1 [v2:172.21.15.78:6810/3035110387,v1:172.21.15.78:6811/3035110387]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-09-16T09:31:12.963 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:31:12 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1[38201]: 2024-09-16T09:31:12.699+0000 7f165d52e700 -1 osd.1 0 waiting for initial osdmap 2024-09-16T09:31:12.963 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:31:12 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1[38201]: 2024-09-16T09:31:12.702+0000 7f1659ec9700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T09:31:13.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:12 smithi088 ceph-mon[29151]: from='osd.1 [v2:172.21.15.78:6810/3035110387,v1:172.21.15.78:6811/3035110387]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-16T09:31:13.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:12 smithi088 ceph-mon[29151]: osdmap e12: 2 total, 1 up, 2 in 2024-09-16T09:31:13.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:12 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T09:31:13.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:12 smithi088 ceph-mon[29151]: from='osd.1 [v2:172.21.15.78:6810/3035110387,v1:172.21.15.78:6811/3035110387]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-09-16T09:31:14.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:13 smithi088 ceph-mon[29151]: from='osd.1 [v2:172.21.15.78:6810/3035110387,v1:172.21.15.78:6811/3035110387]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-09-16T09:31:14.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:13 smithi088 ceph-mon[29151]: osdmap e13: 2 total, 1 up, 2 in 2024-09-16T09:31:14.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:13 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T09:31:14.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:13 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T09:31:14.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:13 smithi088 ceph-mon[29151]: pgmap v53: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:31:14.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:13 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T09:31:14.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:13 smithi078 ceph-mon[23694]: from='osd.1 [v2:172.21.15.78:6810/3035110387,v1:172.21.15.78:6811/3035110387]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-09-16T09:31:14.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:13 smithi078 ceph-mon[23694]: osdmap e13: 2 total, 1 up, 2 in 2024-09-16T09:31:14.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:13 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T09:31:14.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:13 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T09:31:14.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:13 smithi078 ceph-mon[23694]: pgmap v53: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T09:31:14.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:13 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T09:31:14.435 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-16T09:31:14.454 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi078:vg_nvme/lv_2 2024-09-16T09:31:14.991 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:14 smithi078 ceph-mon[23694]: purged_snaps scrub starts 2024-09-16T09:31:14.991 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:14 smithi078 ceph-mon[23694]: purged_snaps scrub ok 2024-09-16T09:31:14.991 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:14 smithi078 ceph-mon[23694]: osd.1 [v2:172.21.15.78:6810/3035110387,v1:172.21.15.78:6811/3035110387] boot 2024-09-16T09:31:14.991 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:14 smithi078 ceph-mon[23694]: osdmap e14: 2 total, 2 up, 2 in 2024-09-16T09:31:14.992 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:14 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T09:31:15.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:14 smithi088 ceph-mon[29151]: purged_snaps scrub starts 2024-09-16T09:31:15.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:14 smithi088 ceph-mon[29151]: purged_snaps scrub ok 2024-09-16T09:31:15.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:14 smithi088 ceph-mon[29151]: osd.1 [v2:172.21.15.78:6810/3035110387,v1:172.21.15.78:6811/3035110387] boot 2024-09-16T09:31:15.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:14 smithi088 ceph-mon[29151]: osdmap e14: 2 total, 2 up, 2 in 2024-09-16T09:31:15.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:14 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T09:31:16.020 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:15 smithi078 ceph-mon[23694]: osdmap e15: 2 total, 2 up, 2 in 2024-09-16T09:31:16.020 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:15 smithi078 ceph-mon[23694]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-16T09:31:16.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:15 smithi088 ceph-mon[29151]: osdmap e15: 2 total, 2 up, 2 in 2024-09-16T09:31:16.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:15 smithi088 ceph-mon[29151]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-16T09:31:18.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:18 smithi088 ceph-mon[29151]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T09:31:18.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:18 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:18.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:18 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T09:31:18.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:18 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T09:31:18.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:18 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:31:18.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:18 smithi078 ceph-mon[23694]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T09:31:18.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:18 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:18.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:18 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T09:31:18.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:18 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T09:31:18.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:18 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:31:19.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:19 smithi078 ceph-mon[23694]: from='client.14322 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:31:19.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:19 smithi088 ceph-mon[29151]: from='client.14322 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:31:20.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:20 smithi078 ceph-mon[23694]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T09:31:20.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:20 smithi088 ceph-mon[29151]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T09:31:22.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:22 smithi078 ceph-mon[23694]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T09:31:22.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:22 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:22.732 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:22 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:31:22.732 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:22 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:31:22.732 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:22 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:22.732 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:22 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:22.732 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:22 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:31:22.732 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:22 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:31:22.732 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:22 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:31:22.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:22 smithi088 ceph-mon[29151]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T09:31:22.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:22 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:22.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:22 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:31:22.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:22 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:31:22.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:22 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:22.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:22 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:22.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:22 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:31:22.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:22 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:31:22.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:22 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:31:23.739 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:23 smithi078 ceph-mon[23694]: Adjusting osd_memory_target on smithi078 to 6432M 2024-09-16T09:31:23.739 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:23 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/757030371' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "08546747-0827-4ff9-b0da-3fbd6d999160"}]: dispatch 2024-09-16T09:31:23.739 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:23 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/757030371' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "08546747-0827-4ff9-b0da-3fbd6d999160"}]': finished 2024-09-16T09:31:23.739 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:23 smithi078 ceph-mon[23694]: osdmap e16: 3 total, 2 up, 3 in 2024-09-16T09:31:23.739 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:23 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T09:31:23.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:23 smithi088 ceph-mon[29151]: Adjusting osd_memory_target on smithi078 to 6432M 2024-09-16T09:31:23.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:23 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/757030371' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "08546747-0827-4ff9-b0da-3fbd6d999160"}]: dispatch 2024-09-16T09:31:23.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:23 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/757030371' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "08546747-0827-4ff9-b0da-3fbd6d999160"}]': finished 2024-09-16T09:31:23.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:23 smithi088 ceph-mon[29151]: osdmap e16: 3 total, 2 up, 3 in 2024-09-16T09:31:23.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:23 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T09:31:24.774 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:24 smithi088 ceph-mon[29151]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T09:31:24.774 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:24 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/2998524317' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T09:31:24.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:24 smithi078 ceph-mon[23694]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T09:31:24.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:24 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/2998524317' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T09:31:26.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:26 smithi088 ceph-mon[29151]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T09:31:26.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:26 smithi078 ceph-mon[23694]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T09:31:28.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:27 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:28.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:27 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:28.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:27 smithi088 ceph-mon[29151]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T09:31:28.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:27 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:28.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:27 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:28.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:27 smithi078 ceph-mon[23694]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T09:31:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:29 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:31:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:29 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/trash_purge_schedule"}]: dispatch 2024-09-16T09:31:29.810 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:29 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:31:29.810 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:29 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/trash_purge_schedule"}]: dispatch 2024-09-16T09:31:30.538 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:30 smithi078 ceph-mon[23694]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T09:31:30.538 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:30 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-16T09:31:30.538 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:30 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:31:30.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:30 smithi088 ceph-mon[29151]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T09:31:30.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:30 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-16T09:31:30.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:30 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:31:31.491 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:31 smithi078 ceph-mon[23694]: Deploying daemon osd.2 on smithi078 2024-09-16T09:31:31.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:31 smithi088 ceph-mon[29151]: Deploying daemon osd.2 on smithi078 2024-09-16T09:31:32.457 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:32 smithi078 ceph-mon[23694]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T09:31:32.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:32 smithi088 ceph-mon[29151]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T09:31:34.550 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:34 smithi088 ceph-mon[29151]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T09:31:34.770 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:34 smithi078 ceph-mon[23694]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T09:31:36.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:36 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:36.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:36 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:31:36.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:36 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:31:36.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:36 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:31:36.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:36 smithi088 ceph-mon[29151]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T09:31:36.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:36 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:36.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:36 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:31:36.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:36 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:31:36.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:36 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:31:36.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:36 smithi078 ceph-mon[23694]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T09:31:38.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:38 smithi078 ceph-mon[23694]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T09:31:38.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:38 smithi078 ceph-mon[23694]: from='osd.2 [v2:172.21.15.78:6818/150643382,v1:172.21.15.78:6819/150643382]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-16T09:31:38.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:38 smithi088 ceph-mon[29151]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T09:31:38.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:38 smithi088 ceph-mon[29151]: from='osd.2 [v2:172.21.15.78:6818/150643382,v1:172.21.15.78:6819/150643382]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-16T09:31:38.953 INFO:teuthology.orchestra.run.smithi078.stdout:Created osd(s) 2 on host 'smithi078' 2024-09-16T09:31:39.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:39 smithi078 ceph-mon[23694]: from='osd.2 [v2:172.21.15.78:6818/150643382,v1:172.21.15.78:6819/150643382]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-16T09:31:39.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:39 smithi078 ceph-mon[23694]: osdmap e17: 3 total, 2 up, 3 in 2024-09-16T09:31:39.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:39 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T09:31:39.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:39 smithi078 ceph-mon[23694]: from='osd.2 [v2:172.21.15.78:6818/150643382,v1:172.21.15.78:6819/150643382]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-09-16T09:31:39.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:39 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:39.589 DEBUG:teuthology.orchestra.run.smithi078:osd.2> sudo journalctl -f -n 0 -u ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.2.service 2024-09-16T09:31:39.591 INFO:tasks.cephadm:Deploying osd.3 on smithi078 with /dev/vg_nvme/lv_1... 2024-09-16T09:31:39.591 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-16T09:31:39.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:39 smithi088 ceph-mon[29151]: from='osd.2 [v2:172.21.15.78:6818/150643382,v1:172.21.15.78:6819/150643382]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-16T09:31:39.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:39 smithi088 ceph-mon[29151]: osdmap e17: 3 total, 2 up, 3 in 2024-09-16T09:31:39.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:39 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T09:31:39.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:39 smithi088 ceph-mon[29151]: from='osd.2 [v2:172.21.15.78:6818/150643382,v1:172.21.15.78:6819/150643382]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-09-16T09:31:39.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:39 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:40.781 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:40 smithi078 ceph-mon[23694]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T09:31:40.781 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:40 smithi078 ceph-mon[23694]: from='osd.2 [v2:172.21.15.78:6818/150643382,v1:172.21.15.78:6819/150643382]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-09-16T09:31:40.781 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:40 smithi078 ceph-mon[23694]: osdmap e18: 3 total, 2 up, 3 in 2024-09-16T09:31:40.781 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:40 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T09:31:40.781 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:40 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T09:31:40.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:40 smithi088 ceph-mon[29151]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T09:31:40.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:40 smithi088 ceph-mon[29151]: from='osd.2 [v2:172.21.15.78:6818/150643382,v1:172.21.15.78:6819/150643382]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-09-16T09:31:40.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:40 smithi088 ceph-mon[29151]: osdmap e18: 3 total, 2 up, 3 in 2024-09-16T09:31:40.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:40 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T09:31:40.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:40 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T09:31:41.822 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:41 smithi078 ceph-mon[23694]: purged_snaps scrub starts 2024-09-16T09:31:41.823 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:41 smithi078 ceph-mon[23694]: purged_snaps scrub ok 2024-09-16T09:31:41.823 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:41 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T09:31:41.823 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:41 smithi078 ceph-mon[23694]: osd.2 [v2:172.21.15.78:6818/150643382,v1:172.21.15.78:6819/150643382] boot 2024-09-16T09:31:41.823 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:41 smithi078 ceph-mon[23694]: osdmap e19: 3 total, 3 up, 3 in 2024-09-16T09:31:41.823 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:41 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T09:31:42.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:41 smithi088 ceph-mon[29151]: purged_snaps scrub starts 2024-09-16T09:31:42.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:41 smithi088 ceph-mon[29151]: purged_snaps scrub ok 2024-09-16T09:31:42.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:41 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T09:31:42.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:41 smithi088 ceph-mon[29151]: osd.2 [v2:172.21.15.78:6818/150643382,v1:172.21.15.78:6819/150643382] boot 2024-09-16T09:31:42.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:41 smithi088 ceph-mon[29151]: osdmap e19: 3 total, 3 up, 3 in 2024-09-16T09:31:42.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:41 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T09:31:42.714 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:42 smithi078 ceph-mon[23694]: pgmap v73: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T09:31:42.714 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:42 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-16T09:31:42.714 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:42 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-16T09:31:42.715 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:42 smithi078 ceph-mon[23694]: osdmap e20: 3 total, 3 up, 3 in 2024-09-16T09:31:42.715 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:42 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-16T09:31:43.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:42 smithi088 ceph-mon[29151]: pgmap v73: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T09:31:43.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:42 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-16T09:31:43.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:42 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-16T09:31:43.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:42 smithi088 ceph-mon[29151]: osdmap e20: 3 total, 3 up, 3 in 2024-09-16T09:31:43.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:42 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-16T09:31:43.643 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:31:43 smithi078 sudo[43240]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-16T09:31:43.644 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:31:43 smithi078 sudo[43240]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-16T09:31:43.644 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:31:43 smithi078 sudo[43240]: pam_unix(sudo:session): session closed for user root 2024-09-16T09:31:43.678 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-16T09:31:43.695 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi078:vg_nvme/lv_1 2024-09-16T09:31:43.922 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:43 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-16T09:31:43.922 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:43 smithi078 ceph-mon[23694]: osdmap e21: 3 total, 3 up, 3 in 2024-09-16T09:31:43.922 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:43 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:43.922 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:43 smithi078 ceph-mon[23694]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-16T09:31:43.922 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:31:43 smithi078 sudo[43260]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-16T09:31:43.922 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:31:43 smithi078 sudo[43260]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-16T09:31:43.923 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:31:43 smithi078 sudo[43260]: pam_unix(sudo:session): session closed for user root 2024-09-16T09:31:44.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:43 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-16T09:31:44.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:43 smithi088 ceph-mon[29151]: osdmap e21: 3 total, 3 up, 3 in 2024-09-16T09:31:44.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:43 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:44.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:43 smithi088 ceph-mon[29151]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-16T09:31:44.187 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:31:43 smithi078 sudo[43333]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-16T09:31:44.187 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:31:43 smithi078 sudo[43333]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-16T09:31:44.187 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:31:43 smithi078 sudo[43333]: pam_unix(sudo:session): session closed for user root 2024-09-16T09:31:44.446 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:31:44 smithi078 sudo[43357]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-16T09:31:44.446 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:31:44 smithi078 sudo[43357]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-16T09:31:44.446 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:31:44 smithi078 sudo[43357]: pam_unix(sudo:session): session closed for user root 2024-09-16T09:31:44.447 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:31:44 smithi078 sudo[43360]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-16T09:31:44.447 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:31:44 smithi078 sudo[43360]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-16T09:31:44.447 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:31:44 smithi078 sudo[43360]: pam_unix(sudo:session): session closed for user root 2024-09-16T09:31:44.725 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:44 smithi078 ceph-mon[23694]: osdmap e22: 3 total, 3 up, 3 in 2024-09-16T09:31:44.725 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:44 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:44.726 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:31:44 smithi078 sudo[43363]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-16T09:31:44.726 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:31:44 smithi078 sudo[43363]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-16T09:31:44.726 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:31:44 smithi078 sudo[43363]: pam_unix(sudo:session): session closed for user root 2024-09-16T09:31:45.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:44 smithi088 ceph-mon[29151]: osdmap e22: 3 total, 3 up, 3 in 2024-09-16T09:31:45.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:44 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:45.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:44 smithi078 sudo[43383]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-16T09:31:45.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:44 smithi078 sudo[43383]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-16T09:31:45.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:44 smithi078 sudo[43383]: pam_unix(sudo:session): session closed for user root 2024-09-16T09:31:45.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:45 smithi078 sudo[43387]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-16T09:31:45.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:45 smithi078 sudo[43387]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-16T09:31:45.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:45 smithi078 sudo[43387]: pam_unix(sudo:session): session closed for user root 2024-09-16T09:31:45.690 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:45 smithi088 sudo[31461]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-16T09:31:45.690 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:45 smithi088 sudo[31461]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-16T09:31:45.690 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:45 smithi088 sudo[31461]: pam_unix(sudo:session): session closed for user root 2024-09-16T09:31:45.690 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:45 smithi088 sudo[31464]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-16T09:31:45.690 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:45 smithi088 sudo[31464]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-16T09:31:45.691 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:45 smithi088 sudo[31464]: pam_unix(sudo:session): session closed for user root 2024-09-16T09:31:45.925 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:45 smithi078 ceph-mon[23694]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-16T09:31:45.926 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:45 smithi078 ceph-mon[23694]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-16T09:31:45.926 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:45 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-16T09:31:45.926 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:45 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi088"}]: dispatch 2024-09-16T09:31:45.926 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:45 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-16T09:31:45.926 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:45 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi088"}]: dispatch 2024-09-16T09:31:45.926 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:45 smithi078 ceph-mon[23694]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-09-16T09:31:46.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:45 smithi088 ceph-mon[29151]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-16T09:31:46.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:45 smithi088 ceph-mon[29151]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-16T09:31:46.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:45 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-16T09:31:46.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:45 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi088"}]: dispatch 2024-09-16T09:31:46.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:45 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-16T09:31:46.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:45 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi088"}]: dispatch 2024-09-16T09:31:46.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:45 smithi088 ceph-mon[29151]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-09-16T09:31:46.888 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:46 smithi078 ceph-mon[23694]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-16T09:31:46.889 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:46 smithi078 ceph-mon[23694]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-16T09:31:46.889 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:46 smithi078 ceph-mon[23694]: mgrmap e19: smithi078.rrfxge(active, since 2m), standbys: smithi088.ncxfdz 2024-09-16T09:31:46.889 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:46 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:46.889 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:46 smithi078 ceph-mon[23694]: from='client.14346 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:31:46.889 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:46 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T09:31:46.889 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:46 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T09:31:46.889 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:46 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:31:47.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:46 smithi088 ceph-mon[29151]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-16T09:31:47.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:46 smithi088 ceph-mon[29151]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-16T09:31:47.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:46 smithi088 ceph-mon[29151]: mgrmap e19: smithi078.rrfxge(active, since 2m), standbys: smithi088.ncxfdz 2024-09-16T09:31:47.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:46 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:47.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:46 smithi088 ceph-mon[29151]: from='client.14346 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:31:47.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:46 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T09:31:47.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:46 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T09:31:47.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:46 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:31:47.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:47 smithi078 ceph-mon[23694]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-16T09:31:48.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:47 smithi088 ceph-mon[29151]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-16T09:31:49.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:49 smithi088 ceph-mon[29151]: Detected new or changed devices on smithi078 2024-09-16T09:31:49.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:49 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:49.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:49 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:31:49.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:49 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:31:49.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:49 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:31:49.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:49 smithi088 ceph-mon[29151]: Adjusting osd_memory_target on smithi078 to 4288M 2024-09-16T09:31:49.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:49 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:49.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:49 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:49.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:49 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:31:49.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:49 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:31:49.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:49 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:31:49.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:49 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:49.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:49 smithi078 ceph-mon[23694]: Detected new or changed devices on smithi078 2024-09-16T09:31:49.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:49 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:49.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:49 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:31:49.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:49 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:31:49.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:49 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:31:49.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:49 smithi078 ceph-mon[23694]: Adjusting osd_memory_target on smithi078 to 4288M 2024-09-16T09:31:49.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:49 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:49.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:49 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:49.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:49 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:31:49.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:49 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:31:49.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:49 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:31:49.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:49 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:31:50.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:50 smithi088 ceph-mon[29151]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-16T09:31:50.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:50 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/957015443' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "17283fa7-427a-4a02-b313-a39a39e9b4e7"}]: dispatch 2024-09-16T09:31:50.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:50 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/957015443' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "17283fa7-427a-4a02-b313-a39a39e9b4e7"}]': finished 2024-09-16T09:31:50.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:50 smithi088 ceph-mon[29151]: osdmap e23: 4 total, 3 up, 4 in 2024-09-16T09:31:50.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:50 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T09:31:50.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:50 smithi078 ceph-mon[23694]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-16T09:31:50.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:50 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/957015443' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "17283fa7-427a-4a02-b313-a39a39e9b4e7"}]: dispatch 2024-09-16T09:31:50.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:50 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/957015443' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "17283fa7-427a-4a02-b313-a39a39e9b4e7"}]': finished 2024-09-16T09:31:50.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:50 smithi078 ceph-mon[23694]: osdmap e23: 4 total, 3 up, 4 in 2024-09-16T09:31:50.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:50 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T09:31:51.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:51 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/3756115145' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T09:31:51.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:51 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/3756115145' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T09:31:52.702 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:52 smithi088 ceph-mon[29151]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-16T09:31:52.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:52 smithi078 ceph-mon[23694]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-16T09:31:54.795 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:54 smithi078 ceph-mon[23694]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-16T09:31:54.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:54 smithi088 ceph-mon[29151]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-16T09:31:56.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:56 smithi078 ceph-mon[23694]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-16T09:31:56.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:56 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-16T09:31:56.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:56 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:31:56.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:56 smithi088 ceph-mon[29151]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-16T09:31:56.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:56 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-16T09:31:56.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:56 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:31:57.750 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:57 smithi078 ceph-mon[23694]: Deploying daemon osd.3 on smithi078 2024-09-16T09:31:57.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:57 smithi088 ceph-mon[29151]: Deploying daemon osd.3 on smithi078 2024-09-16T09:31:58.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:31:58 smithi088 ceph-mon[29151]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-16T09:31:58.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:31:58 smithi078 ceph-mon[23694]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-16T09:32:00.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:00 smithi088 ceph-mon[29151]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-16T09:32:00.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:00 smithi078 ceph-mon[23694]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-16T09:32:02.469 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:02 smithi078 ceph-mon[23694]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-16T09:32:02.469 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:02 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:02.470 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:02 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:32:02.470 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:02 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:32:02.470 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:02 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:32:02.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:02 smithi088 ceph-mon[29151]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-16T09:32:02.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:02 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:02.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:02 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:32:02.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:02 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:32:02.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:02 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:32:04.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:04 smithi088 ceph-mon[29151]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-16T09:32:04.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:04 smithi078 ceph-mon[23694]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-16T09:32:04.885 INFO:teuthology.orchestra.run.smithi078.stdout:Created osd(s) 3 on host 'smithi078' 2024-09-16T09:32:05.856 DEBUG:teuthology.orchestra.run.smithi078:osd.3> sudo journalctl -f -n 0 -u ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.3.service 2024-09-16T09:32:05.859 INFO:tasks.cephadm:Deploying osd.4 on smithi088 with /dev/vg_nvme/lv_4... 2024-09-16T09:32:05.859 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-16T09:32:06.111 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:05 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:06.111 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:05 smithi078 ceph-mon[23694]: from='osd.3 [v2:172.21.15.78:6826/1372470838,v1:172.21.15.78:6827/1372470838]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-16T09:32:06.111 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:05 smithi078 ceph-mon[23694]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-16T09:32:06.165 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:05 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:06.166 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:05 smithi088 ceph-mon[29151]: from='osd.3 [v2:172.21.15.78:6826/1372470838,v1:172.21.15.78:6827/1372470838]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-16T09:32:06.166 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:05 smithi088 ceph-mon[29151]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-16T09:32:07.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:06 smithi078 ceph-mon[23694]: from='osd.3 [v2:172.21.15.78:6826/1372470838,v1:172.21.15.78:6827/1372470838]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-16T09:32:07.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:06 smithi078 ceph-mon[23694]: osdmap e24: 4 total, 3 up, 4 in 2024-09-16T09:32:07.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:06 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T09:32:07.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:06 smithi078 ceph-mon[23694]: from='osd.3 [v2:172.21.15.78:6826/1372470838,v1:172.21.15.78:6827/1372470838]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-09-16T09:32:07.062 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:32:06 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3[45107]: 2024-09-16T09:32:06.892+0000 7f591a1ba700 -1 osd.3 0 waiting for initial osdmap 2024-09-16T09:32:07.062 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:32:06 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3[45107]: 2024-09-16T09:32:06.896+0000 7f5914350700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T09:32:07.182 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:06 smithi088 ceph-mon[29151]: from='osd.3 [v2:172.21.15.78:6826/1372470838,v1:172.21.15.78:6827/1372470838]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-16T09:32:07.182 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:06 smithi088 ceph-mon[29151]: osdmap e24: 4 total, 3 up, 4 in 2024-09-16T09:32:07.182 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:06 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T09:32:07.182 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:06 smithi088 ceph-mon[29151]: from='osd.3 [v2:172.21.15.78:6826/1372470838,v1:172.21.15.78:6827/1372470838]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-09-16T09:32:08.169 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:07 smithi078 ceph-mon[23694]: from='osd.3 [v2:172.21.15.78:6826/1372470838,v1:172.21.15.78:6827/1372470838]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-09-16T09:32:08.169 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:07 smithi078 ceph-mon[23694]: osdmap e25: 4 total, 3 up, 4 in 2024-09-16T09:32:08.169 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:07 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T09:32:08.169 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:07 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T09:32:08.169 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:07 smithi078 ceph-mon[23694]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-16T09:32:08.190 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:07 smithi088 ceph-mon[29151]: from='osd.3 [v2:172.21.15.78:6826/1372470838,v1:172.21.15.78:6827/1372470838]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-09-16T09:32:08.190 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:07 smithi088 ceph-mon[29151]: osdmap e25: 4 total, 3 up, 4 in 2024-09-16T09:32:08.190 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:07 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T09:32:08.190 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:07 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T09:32:08.190 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:07 smithi088 ceph-mon[29151]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-16T09:32:08.454 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:32:08.474 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi088:vg_nvme/lv_4 2024-09-16T09:32:09.035 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:08 smithi088 ceph-mon[29151]: purged_snaps scrub starts 2024-09-16T09:32:09.035 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:08 smithi088 ceph-mon[29151]: purged_snaps scrub ok 2024-09-16T09:32:09.035 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:08 smithi088 ceph-mon[29151]: osd.3 [v2:172.21.15.78:6826/1372470838,v1:172.21.15.78:6827/1372470838] boot 2024-09-16T09:32:09.035 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:08 smithi088 ceph-mon[29151]: osdmap e26: 4 total, 4 up, 4 in 2024-09-16T09:32:09.035 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:08 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T09:32:09.036 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:08 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:09.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:08 smithi078 ceph-mon[23694]: purged_snaps scrub starts 2024-09-16T09:32:09.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:08 smithi078 ceph-mon[23694]: purged_snaps scrub ok 2024-09-16T09:32:09.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:08 smithi078 ceph-mon[23694]: osd.3 [v2:172.21.15.78:6826/1372470838,v1:172.21.15.78:6827/1372470838] boot 2024-09-16T09:32:09.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:08 smithi078 ceph-mon[23694]: osdmap e26: 4 total, 4 up, 4 in 2024-09-16T09:32:09.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:08 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T09:32:09.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:08 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:10.026 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:09 smithi088 ceph-mon[29151]: osdmap e27: 4 total, 4 up, 4 in 2024-09-16T09:32:10.026 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:09 smithi088 ceph-mon[29151]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T09:32:10.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:09 smithi078 ceph-mon[23694]: osdmap e27: 4 total, 4 up, 4 in 2024-09-16T09:32:10.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:09 smithi078 ceph-mon[23694]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T09:32:11.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:10 smithi088 ceph-mon[29151]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi088:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:32:11.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:10 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T09:32:11.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:10 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T09:32:11.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:10 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:32:11.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:10 smithi078 ceph-mon[23694]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi088:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:32:11.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:10 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T09:32:11.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:10 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T09:32:11.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:10 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:32:12.004 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:11 smithi088 ceph-mon[29151]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T09:32:12.220 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:11 smithi078 ceph-mon[23694]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T09:32:13.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:13 smithi088 ceph-mon[29151]: Detected new or changed devices on smithi078 2024-09-16T09:32:13.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:13 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:13.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:13 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:32:13.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:13 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:32:13.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:13 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:32:13.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:13 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:32:13.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:13 smithi088 ceph-mon[29151]: Adjusting osd_memory_target on smithi078 to 3216M 2024-09-16T09:32:13.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:13 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:13.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:13 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:13.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:13 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:32:13.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:13 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:32:13.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:13 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:32:13.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:13 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:13.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:13 smithi088 ceph-mon[29151]: from='client.? 172.21.15.88:0/2560997043' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "21be2fa4-3506-4335-b86c-397b53bb3cc8"}]: dispatch 2024-09-16T09:32:13.553 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:13 smithi088 ceph-mon[29151]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "21be2fa4-3506-4335-b86c-397b53bb3cc8"}]: dispatch 2024-09-16T09:32:13.553 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:13 smithi088 ceph-mon[29151]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "21be2fa4-3506-4335-b86c-397b53bb3cc8"}]': finished 2024-09-16T09:32:13.553 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:13 smithi088 ceph-mon[29151]: osdmap e28: 5 total, 4 up, 5 in 2024-09-16T09:32:13.553 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:13 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T09:32:13.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:13 smithi078 ceph-mon[23694]: Detected new or changed devices on smithi078 2024-09-16T09:32:13.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:13 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:13.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:13 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:32:13.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:13 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:32:13.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:13 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:32:13.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:13 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:32:13.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:13 smithi078 ceph-mon[23694]: Adjusting osd_memory_target on smithi078 to 3216M 2024-09-16T09:32:13.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:13 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:13.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:13 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:13.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:13 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:32:13.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:13 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:32:13.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:13 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:32:13.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:13 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:13.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:13 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/2560997043' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "21be2fa4-3506-4335-b86c-397b53bb3cc8"}]: dispatch 2024-09-16T09:32:13.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:13 smithi078 ceph-mon[23694]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "21be2fa4-3506-4335-b86c-397b53bb3cc8"}]: dispatch 2024-09-16T09:32:13.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:13 smithi078 ceph-mon[23694]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "21be2fa4-3506-4335-b86c-397b53bb3cc8"}]': finished 2024-09-16T09:32:13.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:13 smithi078 ceph-mon[23694]: osdmap e28: 5 total, 4 up, 5 in 2024-09-16T09:32:13.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:13 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T09:32:14.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:14 smithi088 ceph-mon[29151]: from='client.? 172.21.15.88:0/2942934786' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T09:32:14.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:14 smithi088 ceph-mon[29151]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T09:32:14.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:14 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/2942934786' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T09:32:14.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:14 smithi078 ceph-mon[23694]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T09:32:16.531 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:16 smithi088 ceph-mon[29151]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T09:32:16.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:16 smithi078 ceph-mon[23694]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T09:32:18.725 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:18 smithi088 ceph-mon[29151]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T09:32:18.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:18 smithi078 ceph-mon[23694]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T09:32:19.733 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:19 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-16T09:32:19.733 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:19 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:32:19.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:19 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-16T09:32:19.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:19 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:32:20.771 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:20 smithi088 ceph-mon[29151]: Deploying daemon osd.4 on smithi088 2024-09-16T09:32:20.772 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:20 smithi088 ceph-mon[29151]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T09:32:20.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:20 smithi078 ceph-mon[23694]: Deploying daemon osd.4 on smithi088 2024-09-16T09:32:20.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:20 smithi078 ceph-mon[23694]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T09:32:22.549 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:22 smithi088 ceph-mon[29151]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T09:32:22.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:22 smithi078 ceph-mon[23694]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T09:32:24.655 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:24 smithi088 ceph-mon[29151]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T09:32:24.656 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:24.656 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:32:24.656 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:32:24.656 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:32:24.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:24 smithi078 ceph-mon[23694]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T09:32:24.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:24.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:32:24.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:32:24.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:32:26.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:26 smithi088 ceph-mon[29151]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T09:32:26.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:26 smithi078 ceph-mon[23694]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T09:32:27.020 INFO:teuthology.orchestra.run.smithi088.stdout:Created osd(s) 4 on host 'smithi088' 2024-09-16T09:32:27.550 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:27 smithi088 ceph-mon[29151]: from='osd.4 [v2:172.21.15.88:6800/383049394,v1:172.21.15.88:6801/383049394]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-16T09:32:27.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:27 smithi088 ceph-mon[29151]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-16T09:32:27.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:27 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:27.667 DEBUG:teuthology.orchestra.run.smithi088:osd.4> sudo journalctl -f -n 0 -u ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.4.service 2024-09-16T09:32:27.670 INFO:tasks.cephadm:Deploying osd.5 on smithi088 with /dev/vg_nvme/lv_3... 2024-09-16T09:32:27.670 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-16T09:32:27.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:27 smithi078 ceph-mon[23694]: from='osd.4 [v2:172.21.15.88:6800/383049394,v1:172.21.15.88:6801/383049394]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-16T09:32:27.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:27 smithi078 ceph-mon[23694]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-16T09:32:27.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:27 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:28.792 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:28 smithi088 ceph-mon[29151]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T09:32:28.792 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:28 smithi088 ceph-mon[29151]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-16T09:32:28.792 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:28 smithi088 ceph-mon[29151]: osdmap e29: 5 total, 4 up, 5 in 2024-09-16T09:32:28.792 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:28 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T09:32:28.792 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:28 smithi088 ceph-mon[29151]: from='osd.4 [v2:172.21.15.88:6800/383049394,v1:172.21.15.88:6801/383049394]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-16T09:32:28.792 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:28 smithi088 ceph-mon[29151]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-16T09:32:28.792 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:28 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:28.793 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:32:28 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4[33375]: 2024-09-16T09:32:28.447+0000 7fbd057ce700 -1 osd.4 0 waiting for initial osdmap 2024-09-16T09:32:28.793 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:32:28 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4[33375]: 2024-09-16T09:32:28.451+0000 7fbd01167700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T09:32:28.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:28 smithi078 ceph-mon[23694]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T09:32:28.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:28 smithi078 ceph-mon[23694]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-16T09:32:28.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:28 smithi078 ceph-mon[23694]: osdmap e29: 5 total, 4 up, 5 in 2024-09-16T09:32:28.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:28 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T09:32:28.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:28 smithi078 ceph-mon[23694]: from='osd.4 [v2:172.21.15.88:6800/383049394,v1:172.21.15.88:6801/383049394]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-16T09:32:28.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:28 smithi078 ceph-mon[23694]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-16T09:32:28.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:28 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:29.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:29 smithi088 ceph-mon[29151]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]': finished 2024-09-16T09:32:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:29 smithi088 ceph-mon[29151]: osdmap e30: 5 total, 4 up, 5 in 2024-09-16T09:32:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:29 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T09:32:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:29 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T09:32:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:29 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:32:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:29 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/trash_purge_schedule"}]: dispatch 2024-09-16T09:32:29.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:29 smithi078 ceph-mon[23694]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]': finished 2024-09-16T09:32:29.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:29 smithi078 ceph-mon[23694]: osdmap e30: 5 total, 4 up, 5 in 2024-09-16T09:32:29.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:29 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T09:32:29.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:29 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T09:32:29.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:29 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:32:29.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:29 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/trash_purge_schedule"}]: dispatch 2024-09-16T09:32:30.530 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:30 smithi088 ceph-mon[29151]: purged_snaps scrub starts 2024-09-16T09:32:30.530 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:30 smithi088 ceph-mon[29151]: purged_snaps scrub ok 2024-09-16T09:32:30.530 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:30 smithi088 ceph-mon[29151]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T09:32:30.530 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:30 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T09:32:30.530 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:30 smithi088 ceph-mon[29151]: osd.4 [v2:172.21.15.88:6800/383049394,v1:172.21.15.88:6801/383049394] boot 2024-09-16T09:32:30.531 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:30 smithi088 ceph-mon[29151]: osdmap e31: 5 total, 5 up, 5 in 2024-09-16T09:32:30.531 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:30 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T09:32:30.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:30 smithi078 ceph-mon[23694]: purged_snaps scrub starts 2024-09-16T09:32:30.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:30 smithi078 ceph-mon[23694]: purged_snaps scrub ok 2024-09-16T09:32:30.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:30 smithi078 ceph-mon[23694]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T09:32:30.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:30 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T09:32:30.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:30 smithi078 ceph-mon[23694]: osd.4 [v2:172.21.15.88:6800/383049394,v1:172.21.15.88:6801/383049394] boot 2024-09-16T09:32:30.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:30 smithi078 ceph-mon[23694]: osdmap e31: 5 total, 5 up, 5 in 2024-09-16T09:32:30.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:30 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T09:32:31.035 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:32:31.056 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi088:vg_nvme/lv_3 2024-09-16T09:32:31.685 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:31 smithi088 ceph-mon[29151]: osdmap e32: 5 total, 5 up, 5 in 2024-09-16T09:32:31.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:31 smithi078 ceph-mon[23694]: osdmap e32: 5 total, 5 up, 5 in 2024-09-16T09:32:32.784 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:32 smithi088 ceph-mon[29151]: pgmap v111: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-16T09:32:32.784 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:32 smithi088 ceph-mon[29151]: osdmap e33: 5 total, 5 up, 5 in 2024-09-16T09:32:32.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:32 smithi078 ceph-mon[23694]: pgmap v111: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-16T09:32:32.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:32 smithi078 ceph-mon[23694]: osdmap e33: 5 total, 5 up, 5 in 2024-09-16T09:32:33.671 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:33 smithi088 ceph-mon[29151]: osdmap e34: 5 total, 5 up, 5 in 2024-09-16T09:32:33.671 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:33 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:33.671 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:33 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:32:33.671 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:33 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:33.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:33 smithi078 ceph-mon[23694]: osdmap e34: 5 total, 5 up, 5 in 2024-09-16T09:32:33.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:33 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:33.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:33 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:32:33.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:33 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:34.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:34 smithi088 ceph-mon[29151]: pgmap v114: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-16T09:32:34.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:34 smithi088 ceph-mon[29151]: Detected new or changed devices on smithi088 2024-09-16T09:32:34.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:34 smithi088 ceph-mon[29151]: Adjusting osd_memory_target on smithi088 to 15936M 2024-09-16T09:32:34.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:34 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:34.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:34 smithi088 ceph-mon[29151]: osdmap e35: 5 total, 5 up, 5 in 2024-09-16T09:32:34.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:34 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:32:34.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:34 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:32:34.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:34 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:32:34.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:34 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:34.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:34 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T09:32:34.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:34 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T09:32:34.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:34 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:32:34.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:34 smithi078 ceph-mon[23694]: pgmap v114: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-16T09:32:34.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:34 smithi078 ceph-mon[23694]: Detected new or changed devices on smithi088 2024-09-16T09:32:34.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:34 smithi078 ceph-mon[23694]: Adjusting osd_memory_target on smithi088 to 15936M 2024-09-16T09:32:34.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:34 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:34.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:34 smithi078 ceph-mon[23694]: osdmap e35: 5 total, 5 up, 5 in 2024-09-16T09:32:34.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:34 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:32:34.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:34 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:32:34.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:34 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:32:34.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:34 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:34.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:34 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T09:32:34.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:34 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T09:32:34.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:34 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:32:35.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:35 smithi088 ceph-mon[29151]: from='client.24161 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi088:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:32:35.885 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:35 smithi078 ceph-mon[23694]: from='client.24161 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi088:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:32:36.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:36 smithi088 ceph-mon[29151]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 105 KiB/s, 0 objects/s recovering 2024-09-16T09:32:36.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:36 smithi088 ceph-mon[29151]: from='client.? 172.21.15.88:0/2855441407' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4016533c-5627-4110-807c-63c97df53ee2"}]: dispatch 2024-09-16T09:32:36.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:36 smithi088 ceph-mon[29151]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4016533c-5627-4110-807c-63c97df53ee2"}]: dispatch 2024-09-16T09:32:36.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:36 smithi088 ceph-mon[29151]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4016533c-5627-4110-807c-63c97df53ee2"}]': finished 2024-09-16T09:32:36.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:36 smithi088 ceph-mon[29151]: osdmap e36: 6 total, 5 up, 6 in 2024-09-16T09:32:36.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:36 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T09:32:36.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:36 smithi078 ceph-mon[23694]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 105 KiB/s, 0 objects/s recovering 2024-09-16T09:32:36.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:36 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/2855441407' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4016533c-5627-4110-807c-63c97df53ee2"}]: dispatch 2024-09-16T09:32:36.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:36 smithi078 ceph-mon[23694]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4016533c-5627-4110-807c-63c97df53ee2"}]: dispatch 2024-09-16T09:32:36.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:36 smithi078 ceph-mon[23694]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4016533c-5627-4110-807c-63c97df53ee2"}]': finished 2024-09-16T09:32:36.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:36 smithi078 ceph-mon[23694]: osdmap e36: 6 total, 5 up, 6 in 2024-09-16T09:32:36.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:36 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T09:32:37.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:37 smithi088 ceph-mon[29151]: from='client.? 172.21.15.88:0/2810753063' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T09:32:37.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:37 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/2810753063' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T09:32:38.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:38 smithi088 ceph-mon[29151]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-09-16T09:32:38.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:38 smithi078 ceph-mon[23694]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-09-16T09:32:40.758 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:40 smithi088 ceph-mon[29151]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-09-16T09:32:40.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:40 smithi078 ceph-mon[23694]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-09-16T09:32:42.776 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:42 smithi088 ceph-mon[29151]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-16T09:32:42.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:42 smithi078 ceph-mon[23694]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-16T09:32:43.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:43 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-16T09:32:43.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:43 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:32:43.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:43 smithi088 ceph-mon[29151]: Deploying daemon osd.5 on smithi088 2024-09-16T09:32:43.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:43 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-16T09:32:43.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:43 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:32:43.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:43 smithi078 ceph-mon[23694]: Deploying daemon osd.5 on smithi088 2024-09-16T09:32:44.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:44 smithi088 ceph-mon[29151]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-16T09:32:44.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:44 smithi078 ceph-mon[23694]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-16T09:32:46.778 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:46 smithi088 ceph-mon[29151]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-16T09:32:46.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:46 smithi078 ceph-mon[23694]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-16T09:32:48.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:48 smithi088 ceph-mon[29151]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-16T09:32:48.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:48.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:32:48.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:32:48.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:48 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:32:48.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:48 smithi078 ceph-mon[23694]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-16T09:32:48.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:48 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:48.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:48 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:32:48.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:48 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:32:48.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:48 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:32:50.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:50 smithi088 ceph-mon[29151]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-16T09:32:50.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:50 smithi078 ceph-mon[23694]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-16T09:32:51.205 INFO:teuthology.orchestra.run.smithi088.stdout:Created osd(s) 5 on host 'smithi088' 2024-09-16T09:32:51.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:51 smithi078 ceph-mon[23694]: from='osd.5 [v2:172.21.15.88:6808/2909693001,v1:172.21.15.88:6809/2909693001]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-16T09:32:51.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:51 smithi078 ceph-mon[23694]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-16T09:32:51.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:51 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:51.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:51 smithi088 ceph-mon[29151]: from='osd.5 [v2:172.21.15.88:6808/2909693001,v1:172.21.15.88:6809/2909693001]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-16T09:32:51.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:51 smithi088 ceph-mon[29151]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-16T09:32:51.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:51 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:52.278 DEBUG:teuthology.orchestra.run.smithi088:osd.5> sudo journalctl -f -n 0 -u ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.5.service 2024-09-16T09:32:52.281 INFO:tasks.cephadm:Deploying osd.6 on smithi088 with /dev/vg_nvme/lv_2... 2024-09-16T09:32:52.281 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-16T09:32:52.787 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:52 smithi088 ceph-mon[29151]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-16T09:32:52.787 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:52 smithi088 ceph-mon[29151]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-16T09:32:52.787 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:52 smithi088 ceph-mon[29151]: osdmap e37: 6 total, 5 up, 6 in 2024-09-16T09:32:52.787 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:52 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T09:32:52.787 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:52 smithi088 ceph-mon[29151]: from='osd.5 [v2:172.21.15.88:6808/2909693001,v1:172.21.15.88:6809/2909693001]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-16T09:32:52.787 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:52 smithi088 ceph-mon[29151]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-16T09:32:52.787 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:52 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:52.788 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:32:52 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[36324]: 2024-09-16T09:32:52.525+0000 7f1551d35700 -1 osd.5 0 waiting for initial osdmap 2024-09-16T09:32:52.788 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:32:52 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[36324]: 2024-09-16T09:32:52.533+0000 7f154a6c8700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T09:32:52.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:52 smithi078 ceph-mon[23694]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-16T09:32:52.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:52 smithi078 ceph-mon[23694]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-16T09:32:52.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:52 smithi078 ceph-mon[23694]: osdmap e37: 6 total, 5 up, 6 in 2024-09-16T09:32:52.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:52 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T09:32:52.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:52 smithi078 ceph-mon[23694]: from='osd.5 [v2:172.21.15.88:6808/2909693001,v1:172.21.15.88:6809/2909693001]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-16T09:32:52.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:52 smithi078 ceph-mon[23694]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-16T09:32:52.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:52 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:53.796 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:53 smithi088 ceph-mon[29151]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]': finished 2024-09-16T09:32:53.796 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:53 smithi088 ceph-mon[29151]: osdmap e38: 6 total, 5 up, 6 in 2024-09-16T09:32:53.796 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:53 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T09:32:53.797 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:53 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T09:32:53.797 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:53 smithi088 ceph-mon[29151]: osd.5 [v2:172.21.15.88:6808/2909693001,v1:172.21.15.88:6809/2909693001] boot 2024-09-16T09:32:53.797 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:53 smithi088 ceph-mon[29151]: osdmap e39: 6 total, 6 up, 6 in 2024-09-16T09:32:53.797 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:53 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T09:32:53.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:53 smithi078 ceph-mon[23694]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]': finished 2024-09-16T09:32:53.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:53 smithi078 ceph-mon[23694]: osdmap e38: 6 total, 5 up, 6 in 2024-09-16T09:32:53.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:53 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T09:32:53.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:53 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T09:32:53.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:53 smithi078 ceph-mon[23694]: osd.5 [v2:172.21.15.88:6808/2909693001,v1:172.21.15.88:6809/2909693001] boot 2024-09-16T09:32:53.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:53 smithi078 ceph-mon[23694]: osdmap e39: 6 total, 6 up, 6 in 2024-09-16T09:32:53.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:53 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T09:32:54.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:54 smithi088 ceph-mon[29151]: purged_snaps scrub starts 2024-09-16T09:32:54.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:54 smithi088 ceph-mon[29151]: purged_snaps scrub ok 2024-09-16T09:32:54.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:54 smithi088 ceph-mon[29151]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-16T09:32:54.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:54 smithi088 ceph-mon[29151]: osdmap e40: 6 total, 6 up, 6 in 2024-09-16T09:32:54.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:54 smithi078 ceph-mon[23694]: purged_snaps scrub starts 2024-09-16T09:32:54.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:54 smithi078 ceph-mon[23694]: purged_snaps scrub ok 2024-09-16T09:32:54.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:54 smithi078 ceph-mon[23694]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-16T09:32:54.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:54 smithi078 ceph-mon[23694]: osdmap e40: 6 total, 6 up, 6 in 2024-09-16T09:32:55.882 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:32:55.901 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi088:vg_nvme/lv_2 2024-09-16T09:32:56.506 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:56 smithi088 ceph-mon[29151]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-16T09:32:56.506 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:56 smithi088 ceph-mon[29151]: osdmap e41: 6 total, 6 up, 6 in 2024-09-16T09:32:56.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:56 smithi078 ceph-mon[23694]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-16T09:32:56.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:56 smithi078 ceph-mon[23694]: osdmap e41: 6 total, 6 up, 6 in 2024-09-16T09:32:57.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:57 smithi088 ceph-mon[29151]: osdmap e42: 6 total, 6 up, 6 in 2024-09-16T09:32:57.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:57 smithi078 ceph-mon[23694]: osdmap e42: 6 total, 6 up, 6 in 2024-09-16T09:32:58.797 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:58 smithi088 ceph-mon[29151]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-16T09:32:58.798 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:58 smithi088 ceph-mon[29151]: osdmap e43: 6 total, 6 up, 6 in 2024-09-16T09:32:58.798 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:58 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:58.798 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:58 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:32:58.798 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:58 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:32:58.798 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:58 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:58.798 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:58 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:58.798 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:58 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:32:58.798 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:58 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:32:58.798 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:58 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:32:58.798 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:58 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:58.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:58 smithi078 ceph-mon[23694]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-16T09:32:58.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:58 smithi078 ceph-mon[23694]: osdmap e43: 6 total, 6 up, 6 in 2024-09-16T09:32:58.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:58 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:58.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:58 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:32:58.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:58 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:32:58.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:58 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:58.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:58 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:58.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:58 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:32:58.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:58 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:32:58.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:58 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:32:58.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:58 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:32:59.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:59 smithi088 ceph-mon[29151]: Detected new or changed devices on smithi088 2024-09-16T09:32:59.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:59 smithi088 ceph-mon[29151]: Adjusting osd_memory_target on smithi088 to 7968M 2024-09-16T09:32:59.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:59 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T09:32:59.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:59 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T09:32:59.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:32:59 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:32:59.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:59 smithi078 ceph-mon[23694]: Detected new or changed devices on smithi088 2024-09-16T09:32:59.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:59 smithi078 ceph-mon[23694]: Adjusting osd_memory_target on smithi088 to 7968M 2024-09-16T09:32:59.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:59 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T09:32:59.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:59 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T09:32:59.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:32:59 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:33:00.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:00 smithi088 ceph-mon[29151]: from='client.24179 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi088:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:33:00.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:00 smithi088 ceph-mon[29151]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T09:33:00.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:00 smithi078 ceph-mon[23694]: from='client.24179 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi088:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:33:00.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:00 smithi078 ceph-mon[23694]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T09:33:01.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:01 smithi088 ceph-mon[29151]: from='client.? 172.21.15.88:0/1571926575' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f9d66824-5a29-441d-ba52-ba79467980c0"}]: dispatch 2024-09-16T09:33:01.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:01 smithi088 ceph-mon[29151]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f9d66824-5a29-441d-ba52-ba79467980c0"}]: dispatch 2024-09-16T09:33:01.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:01 smithi088 ceph-mon[29151]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f9d66824-5a29-441d-ba52-ba79467980c0"}]': finished 2024-09-16T09:33:01.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:01 smithi088 ceph-mon[29151]: osdmap e44: 7 total, 6 up, 7 in 2024-09-16T09:33:01.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:01 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T09:33:01.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:01 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/1571926575' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f9d66824-5a29-441d-ba52-ba79467980c0"}]: dispatch 2024-09-16T09:33:01.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:01 smithi078 ceph-mon[23694]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f9d66824-5a29-441d-ba52-ba79467980c0"}]: dispatch 2024-09-16T09:33:01.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:01 smithi078 ceph-mon[23694]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f9d66824-5a29-441d-ba52-ba79467980c0"}]': finished 2024-09-16T09:33:01.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:01 smithi078 ceph-mon[23694]: osdmap e44: 7 total, 6 up, 7 in 2024-09-16T09:33:01.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:01 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T09:33:02.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:02 smithi078 ceph-mon[23694]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T09:33:02.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:02 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/3715139606' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T09:33:03.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:02 smithi088 ceph-mon[29151]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T09:33:03.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:02 smithi088 ceph-mon[29151]: from='client.? 172.21.15.88:0/3715139606' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T09:33:04.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:04 smithi088 ceph-mon[29151]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T09:33:04.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:04 smithi078 ceph-mon[23694]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T09:33:06.663 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:06 smithi088 ceph-mon[29151]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T09:33:06.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:06 smithi078 ceph-mon[23694]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T09:33:08.515 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:08 smithi088 ceph-mon[29151]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T09:33:08.515 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:08 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-16T09:33:08.515 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:08 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:33:08.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:08 smithi078 ceph-mon[23694]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T09:33:08.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:08 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-16T09:33:08.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:08 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:33:09.495 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:09 smithi088 ceph-mon[29151]: Deploying daemon osd.6 on smithi088 2024-09-16T09:33:09.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:09 smithi078 ceph-mon[23694]: Deploying daemon osd.6 on smithi088 2024-09-16T09:33:10.531 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:10 smithi088 ceph-mon[29151]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T09:33:10.712 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:10 smithi078 ceph-mon[23694]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T09:33:12.713 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:12 smithi088 ceph-mon[29151]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T09:33:12.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:12 smithi078 ceph-mon[23694]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T09:33:14.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:14 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:33:14.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:14 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:33:14.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:14 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:33:14.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:14 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:33:14.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:14 smithi088 ceph-mon[29151]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T09:33:14.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:14 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:33:14.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:14 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:33:14.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:14 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:33:14.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:14 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:33:14.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:14 smithi078 ceph-mon[23694]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T09:33:16.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:16 smithi088 ceph-mon[29151]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T09:33:16.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:16 smithi088 ceph-mon[29151]: from='osd.6 [v2:172.21.15.88:6816/3916094166,v1:172.21.15.88:6817/3916094166]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-16T09:33:16.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:16 smithi088 ceph-mon[29151]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-16T09:33:16.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:16 smithi078 ceph-mon[23694]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T09:33:16.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:16 smithi078 ceph-mon[23694]: from='osd.6 [v2:172.21.15.88:6816/3916094166,v1:172.21.15.88:6817/3916094166]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-16T09:33:16.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:16 smithi078 ceph-mon[23694]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-16T09:33:16.873 INFO:teuthology.orchestra.run.smithi088.stdout:Created osd(s) 6 on host 'smithi088' 2024-09-16T09:33:17.535 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:17 smithi088 ceph-mon[29151]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-16T09:33:17.535 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:17 smithi088 ceph-mon[29151]: osdmap e45: 7 total, 6 up, 7 in 2024-09-16T09:33:17.535 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:17 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T09:33:17.535 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:17 smithi088 ceph-mon[29151]: from='osd.6 [v2:172.21.15.88:6816/3916094166,v1:172.21.15.88:6817/3916094166]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-16T09:33:17.535 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:17 smithi088 ceph-mon[29151]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-16T09:33:17.535 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:17 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:33:17.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:17 smithi078 ceph-mon[23694]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-16T09:33:17.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:17 smithi078 ceph-mon[23694]: osdmap e45: 7 total, 6 up, 7 in 2024-09-16T09:33:17.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:17 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T09:33:17.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:17 smithi078 ceph-mon[23694]: from='osd.6 [v2:172.21.15.88:6816/3916094166,v1:172.21.15.88:6817/3916094166]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-16T09:33:17.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:17 smithi078 ceph-mon[23694]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-16T09:33:17.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:17 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:33:17.845 DEBUG:teuthology.orchestra.run.smithi088:osd.6> sudo journalctl -f -n 0 -u ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.6.service 2024-09-16T09:33:17.848 INFO:tasks.cephadm:Deploying osd.7 on smithi088 with /dev/vg_nvme/lv_1... 2024-09-16T09:33:17.848 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-16T09:33:18.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:18 smithi088 ceph-mon[29151]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T09:33:18.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:18 smithi088 ceph-mon[29151]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]': finished 2024-09-16T09:33:18.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:18 smithi088 ceph-mon[29151]: osdmap e46: 7 total, 6 up, 7 in 2024-09-16T09:33:18.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:18 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T09:33:18.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:18 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T09:33:18.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:18 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:33:18.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:18 smithi088 ceph-mon[29151]: osd.6 [v2:172.21.15.88:6816/3916094166,v1:172.21.15.88:6817/3916094166] boot 2024-09-16T09:33:18.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:18 smithi088 ceph-mon[29151]: osdmap e47: 7 total, 7 up, 7 in 2024-09-16T09:33:18.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:18 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T09:33:18.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:18 smithi078 ceph-mon[23694]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T09:33:18.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:18 smithi078 ceph-mon[23694]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]': finished 2024-09-16T09:33:18.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:18 smithi078 ceph-mon[23694]: osdmap e46: 7 total, 6 up, 7 in 2024-09-16T09:33:18.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:18 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T09:33:18.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:18 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T09:33:18.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:18 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:33:18.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:18 smithi078 ceph-mon[23694]: osd.6 [v2:172.21.15.88:6816/3916094166,v1:172.21.15.88:6817/3916094166] boot 2024-09-16T09:33:18.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:18 smithi078 ceph-mon[23694]: osdmap e47: 7 total, 7 up, 7 in 2024-09-16T09:33:18.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:18 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T09:33:19.729 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:19 smithi088 ceph-mon[29151]: purged_snaps scrub starts 2024-09-16T09:33:19.729 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:19 smithi088 ceph-mon[29151]: purged_snaps scrub ok 2024-09-16T09:33:19.729 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:19 smithi088 ceph-mon[29151]: osdmap e48: 7 total, 7 up, 7 in 2024-09-16T09:33:19.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:19 smithi078 ceph-mon[23694]: purged_snaps scrub starts 2024-09-16T09:33:19.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:19 smithi078 ceph-mon[23694]: purged_snaps scrub ok 2024-09-16T09:33:19.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:19 smithi078 ceph-mon[23694]: osdmap e48: 7 total, 7 up, 7 in 2024-09-16T09:33:20.619 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:20 smithi088 ceph-mon[29151]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-16T09:33:20.619 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:20 smithi088 ceph-mon[29151]: osdmap e49: 7 total, 7 up, 7 in 2024-09-16T09:33:20.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:20 smithi078 ceph-mon[23694]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-16T09:33:20.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:20 smithi078 ceph-mon[23694]: osdmap e49: 7 total, 7 up, 7 in 2024-09-16T09:33:21.637 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-16T09:33:21.662 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi088:vg_nvme/lv_1 2024-09-16T09:33:22.512 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:22 smithi088 ceph-mon[29151]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-16T09:33:22.512 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:22 smithi088 ceph-mon[29151]: osdmap e50: 7 total, 7 up, 7 in 2024-09-16T09:33:22.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:22 smithi078 ceph-mon[23694]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-16T09:33:22.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:22 smithi078 ceph-mon[23694]: osdmap e50: 7 total, 7 up, 7 in 2024-09-16T09:33:23.553 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:23 smithi088 ceph-mon[29151]: osdmap e51: 7 total, 7 up, 7 in 2024-09-16T09:33:23.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:23 smithi078 ceph-mon[23694]: osdmap e51: 7 total, 7 up, 7 in 2024-09-16T09:33:24.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:24 smithi088 ceph-mon[29151]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T09:33:24.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:33:24.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:33:24.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:33:24.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:33:24.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:33:24.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:33:24.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:33:24.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:33:24.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:33:24.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:33:24.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:24 smithi078 ceph-mon[23694]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T09:33:24.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:33:24.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:33:24.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:33:24.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:33:24.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:33:24.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:33:24.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:33:24.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:33:24.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:33:24.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:33:25.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:25 smithi088 ceph-mon[29151]: Detected new or changed devices on smithi088 2024-09-16T09:33:25.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:25 smithi088 ceph-mon[29151]: Adjusting osd_memory_target on smithi088 to 5312M 2024-09-16T09:33:25.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:25 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T09:33:25.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:25 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T09:33:25.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:25 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:33:25.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:25 smithi078 ceph-mon[23694]: Detected new or changed devices on smithi088 2024-09-16T09:33:25.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:25 smithi078 ceph-mon[23694]: Adjusting osd_memory_target on smithi088 to 5312M 2024-09-16T09:33:25.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:25 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T09:33:25.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:25 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T09:33:25.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:25 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:33:26.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:26 smithi088 ceph-mon[29151]: from='client.24197 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi088:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:33:26.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:26 smithi088 ceph-mon[29151]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-16T09:33:26.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:26 smithi078 ceph-mon[23694]: from='client.24197 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi088:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:33:26.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:26 smithi078 ceph-mon[23694]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-16T09:33:27.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:27 smithi088 ceph-mon[29151]: from='client.? 172.21.15.88:0/2684044136' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5fdda449-e4d6-4e25-985f-b5b57d4eaed2"}]: dispatch 2024-09-16T09:33:27.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:27 smithi088 ceph-mon[29151]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5fdda449-e4d6-4e25-985f-b5b57d4eaed2"}]: dispatch 2024-09-16T09:33:27.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:27 smithi088 ceph-mon[29151]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5fdda449-e4d6-4e25-985f-b5b57d4eaed2"}]': finished 2024-09-16T09:33:27.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:27 smithi088 ceph-mon[29151]: osdmap e52: 8 total, 7 up, 8 in 2024-09-16T09:33:27.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:27 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T09:33:27.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:27 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/2684044136' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5fdda449-e4d6-4e25-985f-b5b57d4eaed2"}]: dispatch 2024-09-16T09:33:27.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:27 smithi078 ceph-mon[23694]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5fdda449-e4d6-4e25-985f-b5b57d4eaed2"}]: dispatch 2024-09-16T09:33:27.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:27 smithi078 ceph-mon[23694]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5fdda449-e4d6-4e25-985f-b5b57d4eaed2"}]': finished 2024-09-16T09:33:27.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:27 smithi078 ceph-mon[23694]: osdmap e52: 8 total, 7 up, 8 in 2024-09-16T09:33:27.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:27 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T09:33:28.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:28 smithi088 ceph-mon[29151]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-16T09:33:28.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:28 smithi088 ceph-mon[29151]: from='client.? 172.21.15.88:0/343434759' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T09:33:28.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:28 smithi078 ceph-mon[23694]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-16T09:33:28.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:28 smithi078 ceph-mon[23694]: from='client.? 172.21.15.88:0/343434759' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T09:33:29.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:29 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:33:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:29 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/trash_purge_schedule"}]: dispatch 2024-09-16T09:33:29.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:29 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:33:29.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:29 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/trash_purge_schedule"}]: dispatch 2024-09-16T09:33:30.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:30 smithi088 ceph-mon[29151]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-16T09:33:30.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:30 smithi078 ceph-mon[23694]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-16T09:33:32.776 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:32 smithi088 ceph-mon[29151]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-16T09:33:32.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:32 smithi078 ceph-mon[23694]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-16T09:33:34.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:34 smithi088 ceph-mon[29151]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-16T09:33:34.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:34 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-16T09:33:34.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:34 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:33:34.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:34 smithi088 ceph-mon[29151]: Deploying daemon osd.7 on smithi088 2024-09-16T09:33:34.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:34 smithi078 ceph-mon[23694]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-16T09:33:34.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:34 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-16T09:33:34.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:34 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:33:34.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:34 smithi078 ceph-mon[23694]: Deploying daemon osd.7 on smithi088 2024-09-16T09:33:36.674 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:36 smithi088 ceph-mon[29151]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T09:33:36.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:36 smithi078 ceph-mon[23694]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T09:33:38.645 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:38 smithi088 ceph-mon[29151]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T09:33:38.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:38 smithi078 ceph-mon[23694]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T09:33:40.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:40 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:33:40.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:40 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:33:40.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:40 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:33:40.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:40 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:33:40.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:40 smithi088 ceph-mon[29151]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T09:33:40.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:40 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:33:40.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:40 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:33:40.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:40 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:33:40.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:40 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:33:40.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:40 smithi078 ceph-mon[23694]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T09:33:42.557 INFO:teuthology.orchestra.run.smithi088.stdout:Created osd(s) 7 on host 'smithi088' 2024-09-16T09:33:42.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:42 smithi088 ceph-mon[29151]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T09:33:42.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:42 smithi088 ceph-mon[29151]: from='osd.7 [v2:172.21.15.88:6824/897007524,v1:172.21.15.88:6825/897007524]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-16T09:33:42.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:42 smithi088 ceph-mon[29151]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-16T09:33:42.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:42 smithi078 ceph-mon[23694]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T09:33:42.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:42 smithi078 ceph-mon[23694]: from='osd.7 [v2:172.21.15.88:6824/897007524,v1:172.21.15.88:6825/897007524]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-16T09:33:42.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:42 smithi078 ceph-mon[23694]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-16T09:33:43.512 DEBUG:teuthology.orchestra.run.smithi088:osd.7> sudo journalctl -f -n 0 -u ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.7.service 2024-09-16T09:33:43.515 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-16T09:33:43.515 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-16T09:33:43.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:43 smithi088 ceph-mon[29151]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-16T09:33:43.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:43 smithi088 ceph-mon[29151]: osdmap e53: 8 total, 7 up, 8 in 2024-09-16T09:33:43.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:43 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T09:33:43.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:43 smithi088 ceph-mon[29151]: from='osd.7 [v2:172.21.15.88:6824/897007524,v1:172.21.15.88:6825/897007524]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-16T09:33:43.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:43 smithi088 ceph-mon[29151]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-16T09:33:43.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:43 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:33:43.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:43 smithi078 ceph-mon[23694]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-16T09:33:43.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:43 smithi078 ceph-mon[23694]: osdmap e53: 8 total, 7 up, 8 in 2024-09-16T09:33:43.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:43 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T09:33:43.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:43 smithi078 ceph-mon[23694]: from='osd.7 [v2:172.21.15.88:6824/897007524,v1:172.21.15.88:6825/897007524]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-16T09:33:43.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:43 smithi078 ceph-mon[23694]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-16T09:33:43.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:43 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:33:44.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:44 smithi088 ceph-mon[29151]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T09:33:44.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:44 smithi088 ceph-mon[29151]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]': finished 2024-09-16T09:33:44.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:44 smithi088 ceph-mon[29151]: osdmap e54: 8 total, 7 up, 8 in 2024-09-16T09:33:44.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:44 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T09:33:44.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:44 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T09:33:44.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:44 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:33:44.789 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:44 smithi078 ceph-mon[23694]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T09:33:44.789 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:44 smithi078 ceph-mon[23694]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]': finished 2024-09-16T09:33:44.789 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:44 smithi078 ceph-mon[23694]: osdmap e54: 8 total, 7 up, 8 in 2024-09-16T09:33:44.789 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:44 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T09:33:44.789 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:44 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T09:33:44.789 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:44 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:33:45.418 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-16T09:33:45.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:45 smithi088 ceph-mon[29151]: purged_snaps scrub starts 2024-09-16T09:33:45.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:45 smithi088 ceph-mon[29151]: purged_snaps scrub ok 2024-09-16T09:33:45.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:45 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T09:33:45.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:45 smithi088 ceph-mon[29151]: osd.7 [v2:172.21.15.88:6824/897007524,v1:172.21.15.88:6825/897007524] boot 2024-09-16T09:33:45.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:45 smithi088 ceph-mon[29151]: osdmap e55: 8 total, 8 up, 8 in 2024-09-16T09:33:45.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:45 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T09:33:45.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:45 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/4119337676' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-16T09:33:45.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:45 smithi078 ceph-mon[23694]: purged_snaps scrub starts 2024-09-16T09:33:45.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:45 smithi078 ceph-mon[23694]: purged_snaps scrub ok 2024-09-16T09:33:45.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:45 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T09:33:45.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:45 smithi078 ceph-mon[23694]: osd.7 [v2:172.21.15.88:6824/897007524,v1:172.21.15.88:6825/897007524] boot 2024-09-16T09:33:45.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:45 smithi078 ceph-mon[23694]: osdmap e55: 8 total, 8 up, 8 in 2024-09-16T09:33:45.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:45 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T09:33:45.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:45 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/4119337676' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-16T09:33:45.928 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1726479224,"num_in_osds":8,"osd_in_since":1726479207,"num_remapped_pgs":1} 2024-09-16T09:33:45.928 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-16T09:33:46.133 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-16T09:33:46.548 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:46 smithi078 ceph-mon[23694]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T09:33:46.548 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:46 smithi078 ceph-mon[23694]: osdmap e56: 8 total, 8 up, 8 in 2024-09-16T09:33:46.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:46 smithi088 ceph-mon[29151]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T09:33:46.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:46 smithi088 ceph-mon[29151]: osdmap e56: 8 total, 8 up, 8 in 2024-09-16T09:33:47.555 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:47 smithi078 ceph-mon[23694]: osdmap e57: 8 total, 8 up, 8 in 2024-09-16T09:33:47.555 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:47 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:33:47.555 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:47 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:33:47.555 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:47 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:33:47.555 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:47 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:33:47.555 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:47 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:33:47.555 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:47 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:33:47.555 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:47 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:33:47.555 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:47 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:33:47.555 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:47 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:33:47.555 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:47 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:33:47.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:47 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:33:47.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:47 smithi088 ceph-mon[29151]: osdmap e57: 8 total, 8 up, 8 in 2024-09-16T09:33:47.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:47 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:33:47.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:47 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:33:47.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:47 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:33:47.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:47 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:33:47.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:47 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:33:47.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:47 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:33:47.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:47 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:33:47.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:47 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:33:47.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:47 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:33:47.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:47 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:33:47.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:47 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:33:47.834 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-16T09:33:47.834 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":58,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","created":"2024-09-16T09:26:11.529941+0000","modified":"2024-09-16T09:33:47.459394+0000","last_up_change":"2024-09-16T09:33:44.443556+0000","last_in_change":"2024-09-16T09:33:27.045434+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-16T09:31:41.644068+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"9c561ce9-358c-4e1f-aaa0-497e8410c604","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":3273201876},{"type":"v1","addr":"172.21.15.78:6803","nonce":3273201876}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":3273201876},{"type":"v1","addr":"172.21.15.78:6805","nonce":3273201876}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":3273201876},{"type":"v1","addr":"172.21.15.78:6809","nonce":3273201876}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":3273201876},{"type":"v1","addr":"172.21.15.78:6807","nonce":3273201876}]},"public_addr":"172.21.15.78:6803/3273201876","cluster_addr":"172.21.15.78:6805/3273201876","heartbeat_back_addr":"172.21.15.78:6809/3273201876","heartbeat_front_addr":"172.21.15.78:6807/3273201876","state":["exists","up"]},{"osd":1,"uuid":"0af2d2ef-0835-4383-add0-fc6a751b627f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":3035110387},{"type":"v1","addr":"172.21.15.78:6811","nonce":3035110387}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":3035110387},{"type":"v1","addr":"172.21.15.78:6813","nonce":3035110387}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":3035110387},{"type":"v1","addr":"172.21.15.78:6817","nonce":3035110387}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":3035110387},{"type":"v1","addr":"172.21.15.78:6815","nonce":3035110387}]},"public_addr":"172.21.15.78:6811/3035110387","cluster_addr":"172.21.15.78:6813/3035110387","heartbeat_back_addr":"172.21.15.78:6817/3035110387","heartbeat_front_addr":"172.21.15.78:6815/3035110387","state":["exists","up"]},{"osd":2,"uuid":"08546747-0827-4ff9-b0da-3fbd6d999160","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":150643382},{"type":"v1","addr":"172.21.15.78:6819","nonce":150643382}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":150643382},{"type":"v1","addr":"172.21.15.78:6821","nonce":150643382}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6824","nonce":150643382},{"type":"v1","addr":"172.21.15.78:6825","nonce":150643382}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":150643382},{"type":"v1","addr":"172.21.15.78:6823","nonce":150643382}]},"public_addr":"172.21.15.78:6819/150643382","cluster_addr":"172.21.15.78:6821/150643382","heartbeat_back_addr":"172.21.15.78:6825/150643382","heartbeat_front_addr":"172.21.15.78:6823/150643382","state":["exists","up"]},{"osd":3,"uuid":"17283fa7-427a-4a02-b313-a39a39e9b4e7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6826","nonce":1372470838},{"type":"v1","addr":"172.21.15.78:6827","nonce":1372470838}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6828","nonce":1372470838},{"type":"v1","addr":"172.21.15.78:6829","nonce":1372470838}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6832","nonce":1372470838},{"type":"v1","addr":"172.21.15.78:6833","nonce":1372470838}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6830","nonce":1372470838},{"type":"v1","addr":"172.21.15.78:6831","nonce":1372470838}]},"public_addr":"172.21.15.78:6827/1372470838","cluster_addr":"172.21.15.78:6829/1372470838","heartbeat_back_addr":"172.21.15.78:6833/1372470838","heartbeat_front_addr":"172.21.15.78:6831/1372470838","state":["exists","up"]},{"osd":4,"uuid":"21be2fa4-3506-4335-b86c-397b53bb3cc8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6800","nonce":383049394},{"type":"v1","addr":"172.21.15.88:6801","nonce":383049394}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6802","nonce":383049394},{"type":"v1","addr":"172.21.15.88:6803","nonce":383049394}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6806","nonce":383049394},{"type":"v1","addr":"172.21.15.88:6807","nonce":383049394}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6804","nonce":383049394},{"type":"v1","addr":"172.21.15.88:6805","nonce":383049394}]},"public_addr":"172.21.15.88:6801/383049394","cluster_addr":"172.21.15.88:6803/383049394","heartbeat_back_addr":"172.21.15.88:6807/383049394","heartbeat_front_addr":"172.21.15.88:6805/383049394","state":["exists","up"]},{"osd":5,"uuid":"4016533c-5627-4110-807c-63c97df53ee2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6808","nonce":2909693001},{"type":"v1","addr":"172.21.15.88:6809","nonce":2909693001}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6810","nonce":2909693001},{"type":"v1","addr":"172.21.15.88:6811","nonce":2909693001}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6814","nonce":2909693001},{"type":"v1","addr":"172.21.15.88:6815","nonce":2909693001}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6812","nonce":2909693001},{"type":"v1","addr":"172.21.15.88:6813","nonce":2909693001}]},"public_addr":"172.21.15.88:6809/2909693001","cluster_addr":"172.21.15.88:6811/2909693001","heartbeat_back_addr":"172.21.15.88:6815/2909693001","heartbeat_front_addr":"172.21.15.88:6813/2909693001","state":["exists","up"]},{"osd":6,"uuid":"f9d66824-5a29-441d-ba52-ba79467980c0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6816","nonce":3916094166},{"type":"v1","addr":"172.21.15.88:6817","nonce":3916094166}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6818","nonce":3916094166},{"type":"v1","addr":"172.21.15.88:6819","nonce":3916094166}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6822","nonce":3916094166},{"type":"v1","addr":"172.21.15.88:6823","nonce":3916094166}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6820","nonce":3916094166},{"type":"v1","addr":"172.21.15.88:6821","nonce":3916094166}]},"public_addr":"172.21.15.88:6817/3916094166","cluster_addr":"172.21.15.88:6819/3916094166","heartbeat_back_addr":"172.21.15.88:6823/3916094166","heartbeat_front_addr":"172.21.15.88:6821/3916094166","state":["exists","up"]},{"osd":7,"uuid":"5fdda449-e4d6-4e25-985f-b5b57d4eaed2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6824","nonce":897007524},{"type":"v1","addr":"172.21.15.88:6825","nonce":897007524}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6826","nonce":897007524},{"type":"v1","addr":"172.21.15.88:6827","nonce":897007524}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6830","nonce":897007524},{"type":"v1","addr":"172.21.15.88:6831","nonce":897007524}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6828","nonce":897007524},{"type":"v1","addr":"172.21.15.88:6829","nonce":897007524}]},"public_addr":"172.21.15.88:6825/897007524","cluster_addr":"172.21.15.88:6827/897007524","heartbeat_back_addr":"172.21.15.88:6831/897007524","heartbeat_front_addr":"172.21.15.88:6829/897007524","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T09:30:38.737600+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T09:31:12.091839+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T09:31:39.117143+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T09:32:06.012629+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T09:32:27.785201+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T09:32:51.896300+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T09:33:17.182195+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T09:33:43.138963+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.78:6800/3897938674":"2024-09-17T09:29:28.526174+0000","172.21.15.78:0/1692915456":"2024-09-17T09:29:28.526174+0000","172.21.15.78:6800/2118705212":"2024-09-17T09:27:13.010642+0000","172.21.15.78:6801/3897938674":"2024-09-17T09:29:28.526174+0000","172.21.15.78:0/1394375203":"2024-09-17T09:26:43.343502+0000","172.21.15.78:0/771928015":"2024-09-17T09:29:28.526174+0000","172.21.15.78:0/1889899684":"2024-09-17T09:27:13.010642+0000","172.21.15.78:0/4262597131":"2024-09-17T09:27:13.010642+0000","172.21.15.78:0/2833844034":"2024-09-17T09:26:43.343502+0000","172.21.15.78:0/3399595328":"2024-09-17T09:27:13.010642+0000","172.21.15.78:6801/2118705212":"2024-09-17T09:27:13.010642+0000","172.21.15.78:6800/2084930300":"2024-09-17T09:26:43.343502+0000","172.21.15.78:0/3767531392":"2024-09-17T09:26:43.343502+0000","172.21.15.78:0/1416709728":"2024-09-17T09:29:28.526174+0000","172.21.15.78:6801/2084930300":"2024-09-17T09:26:43.343502+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-16T09:33:48.449 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-16T09:31:41.644068+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-16T09:33:48.450 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-09-16T09:33:48.659 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-16T09:33:48.780 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:48 smithi078 ceph-mon[23694]: Detected new or changed devices on smithi088 2024-09-16T09:33:48.780 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:48 smithi078 ceph-mon[23694]: Adjusting osd_memory_target on smithi088 to 3984M 2024-09-16T09:33:48.781 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:48 smithi078 ceph-mon[23694]: pgmap v174: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-16T09:33:48.781 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:48 smithi078 ceph-mon[23694]: osdmap e58: 8 total, 8 up, 8 in 2024-09-16T09:33:48.781 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:48 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/1422785548' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-16T09:33:48.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:48 smithi088 ceph-mon[29151]: Detected new or changed devices on smithi088 2024-09-16T09:33:48.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:48 smithi088 ceph-mon[29151]: Adjusting osd_memory_target on smithi088 to 3984M 2024-09-16T09:33:48.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:48 smithi088 ceph-mon[29151]: pgmap v174: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-16T09:33:48.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:48 smithi088 ceph-mon[29151]: osdmap e58: 8 total, 8 up, 8 in 2024-09-16T09:33:48.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:48 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/1422785548' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-16T09:33:49.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:49 smithi088 ceph-mon[29151]: osdmap e59: 8 total, 8 up, 8 in 2024-09-16T09:33:49.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:49 smithi078 ceph-mon[23694]: osdmap e59: 8 total, 8 up, 8 in 2024-09-16T09:33:50.445 INFO:teuthology.orchestra.run.smithi078.stdout:pg_num: 1 2024-09-16T09:33:50.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:50 smithi088 ceph-mon[29151]: pgmap v177: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-16T09:33:50.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:50 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/1115882786' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-16T09:33:50.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:50 smithi078 ceph-mon[23694]: pgmap v177: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-16T09:33:50.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:50 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/1115882786' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-16T09:33:51.038 INFO:tasks.cephadm:Setting up client nodes... 2024-09-16T09:33:51.039 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-16T09:33:52.512 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:52 smithi078 ceph-mon[23694]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-09-16T09:33:52.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:52 smithi088 ceph-mon[29151]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-09-16T09:33:52.971 INFO:teuthology.orchestra.run.smithi078.stdout:[client.0] 2024-09-16T09:33:52.972 INFO:teuthology.orchestra.run.smithi078.stdout: key = AQCA++dmR7yQORAA8KyZX76HETrvLA3kOktBeA== 2024-09-16T09:33:53.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:53 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/3307110192' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-16T09:33:53.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:53 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/3307110192' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-16T09:33:53.586 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-16T09:33:53.586 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-16T09:33:53.587 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-16T09:33:53.624 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph config log 1 --format=json 2024-09-16T09:33:53.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:53 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/3307110192' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-16T09:33:53.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:53 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/3307110192' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-16T09:33:53.870 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-16T09:33:54.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:54 smithi078 ceph-mon[23694]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-09-16T09:33:54.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:54 smithi088 ceph-mon[29151]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-09-16T09:33:55.582 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-16T09:33:56.132 INFO:teuthology.orchestra.run.smithi078.stdout:[{"version":21,"timestamp":"2024-09-16T09:33:46.879648+0000","name":"","changes":[{"name":"osd/host:smithi088/osd_memory_target","previous_value":"5570075033","new_value":"4177556275"}]}] 2024-09-16T09:33:56.133 INFO:tasks.ceph_manager:config epoch is 21 2024-09-16T09:33:56.133 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-16T09:33:56.133 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-16T09:33:56.133 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-16T09:33:56.341 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-16T09:33:56.712 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:56 smithi078 ceph-mon[23694]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:33:56.713 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:56 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/1143133944' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-16T09:33:56.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:56 smithi088 ceph-mon[29151]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:33:56.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:56 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/1143133944' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-16T09:33:58.120 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-16T09:33:58.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:58 smithi088 ceph-mon[29151]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:33:58.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:33:58 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/4043058588' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-16T09:33:58.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:58 smithi078 ceph-mon[23694]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:33:58.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:33:58 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/4043058588' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-16T09:33:58.836 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":19,"active_gid":14252,"active_name":"smithi078.rrfxge","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6800","nonce":2704447453},{"type":"v1","addr":"172.21.15.78:6801","nonce":2704447453}]},"active_addr":"172.21.15.78:6801/2704447453","active_change":"2024-09-16T09:29:28.526529+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14272,"name":"smithi088.ncxfdz","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.78:8443/","prometheus":"http://172.21.15.78:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":307046709}]},{"addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":3406691833}]},{"addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":3053477035}]},{"addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":2234546652}]}]}} 2024-09-16T09:33:58.842 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-16T09:33:58.842 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-16T09:33:58.843 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-16T09:33:59.049 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-16T09:34:00.748 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-16T09:34:00.748 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":59,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","created":"2024-09-16T09:26:11.529941+0000","modified":"2024-09-16T09:33:48.465837+0000","last_up_change":"2024-09-16T09:33:44.443556+0000","last_in_change":"2024-09-16T09:33:27.045434+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-16T09:31:41.644068+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"9c561ce9-358c-4e1f-aaa0-497e8410c604","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":3273201876},{"type":"v1","addr":"172.21.15.78:6803","nonce":3273201876}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":3273201876},{"type":"v1","addr":"172.21.15.78:6805","nonce":3273201876}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":3273201876},{"type":"v1","addr":"172.21.15.78:6809","nonce":3273201876}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":3273201876},{"type":"v1","addr":"172.21.15.78:6807","nonce":3273201876}]},"public_addr":"172.21.15.78:6803/3273201876","cluster_addr":"172.21.15.78:6805/3273201876","heartbeat_back_addr":"172.21.15.78:6809/3273201876","heartbeat_front_addr":"172.21.15.78:6807/3273201876","state":["exists","up"]},{"osd":1,"uuid":"0af2d2ef-0835-4383-add0-fc6a751b627f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":3035110387},{"type":"v1","addr":"172.21.15.78:6811","nonce":3035110387}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":3035110387},{"type":"v1","addr":"172.21.15.78:6813","nonce":3035110387}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":3035110387},{"type":"v1","addr":"172.21.15.78:6817","nonce":3035110387}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":3035110387},{"type":"v1","addr":"172.21.15.78:6815","nonce":3035110387}]},"public_addr":"172.21.15.78:6811/3035110387","cluster_addr":"172.21.15.78:6813/3035110387","heartbeat_back_addr":"172.21.15.78:6817/3035110387","heartbeat_front_addr":"172.21.15.78:6815/3035110387","state":["exists","up"]},{"osd":2,"uuid":"08546747-0827-4ff9-b0da-3fbd6d999160","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":150643382},{"type":"v1","addr":"172.21.15.78:6819","nonce":150643382}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":150643382},{"type":"v1","addr":"172.21.15.78:6821","nonce":150643382}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6824","nonce":150643382},{"type":"v1","addr":"172.21.15.78:6825","nonce":150643382}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":150643382},{"type":"v1","addr":"172.21.15.78:6823","nonce":150643382}]},"public_addr":"172.21.15.78:6819/150643382","cluster_addr":"172.21.15.78:6821/150643382","heartbeat_back_addr":"172.21.15.78:6825/150643382","heartbeat_front_addr":"172.21.15.78:6823/150643382","state":["exists","up"]},{"osd":3,"uuid":"17283fa7-427a-4a02-b313-a39a39e9b4e7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6826","nonce":1372470838},{"type":"v1","addr":"172.21.15.78:6827","nonce":1372470838}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6828","nonce":1372470838},{"type":"v1","addr":"172.21.15.78:6829","nonce":1372470838}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6832","nonce":1372470838},{"type":"v1","addr":"172.21.15.78:6833","nonce":1372470838}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6830","nonce":1372470838},{"type":"v1","addr":"172.21.15.78:6831","nonce":1372470838}]},"public_addr":"172.21.15.78:6827/1372470838","cluster_addr":"172.21.15.78:6829/1372470838","heartbeat_back_addr":"172.21.15.78:6833/1372470838","heartbeat_front_addr":"172.21.15.78:6831/1372470838","state":["exists","up"]},{"osd":4,"uuid":"21be2fa4-3506-4335-b86c-397b53bb3cc8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6800","nonce":383049394},{"type":"v1","addr":"172.21.15.88:6801","nonce":383049394}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6802","nonce":383049394},{"type":"v1","addr":"172.21.15.88:6803","nonce":383049394}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6806","nonce":383049394},{"type":"v1","addr":"172.21.15.88:6807","nonce":383049394}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6804","nonce":383049394},{"type":"v1","addr":"172.21.15.88:6805","nonce":383049394}]},"public_addr":"172.21.15.88:6801/383049394","cluster_addr":"172.21.15.88:6803/383049394","heartbeat_back_addr":"172.21.15.88:6807/383049394","heartbeat_front_addr":"172.21.15.88:6805/383049394","state":["exists","up"]},{"osd":5,"uuid":"4016533c-5627-4110-807c-63c97df53ee2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6808","nonce":2909693001},{"type":"v1","addr":"172.21.15.88:6809","nonce":2909693001}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6810","nonce":2909693001},{"type":"v1","addr":"172.21.15.88:6811","nonce":2909693001}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6814","nonce":2909693001},{"type":"v1","addr":"172.21.15.88:6815","nonce":2909693001}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6812","nonce":2909693001},{"type":"v1","addr":"172.21.15.88:6813","nonce":2909693001}]},"public_addr":"172.21.15.88:6809/2909693001","cluster_addr":"172.21.15.88:6811/2909693001","heartbeat_back_addr":"172.21.15.88:6815/2909693001","heartbeat_front_addr":"172.21.15.88:6813/2909693001","state":["exists","up"]},{"osd":6,"uuid":"f9d66824-5a29-441d-ba52-ba79467980c0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6816","nonce":3916094166},{"type":"v1","addr":"172.21.15.88:6817","nonce":3916094166}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6818","nonce":3916094166},{"type":"v1","addr":"172.21.15.88:6819","nonce":3916094166}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6822","nonce":3916094166},{"type":"v1","addr":"172.21.15.88:6823","nonce":3916094166}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6820","nonce":3916094166},{"type":"v1","addr":"172.21.15.88:6821","nonce":3916094166}]},"public_addr":"172.21.15.88:6817/3916094166","cluster_addr":"172.21.15.88:6819/3916094166","heartbeat_back_addr":"172.21.15.88:6823/3916094166","heartbeat_front_addr":"172.21.15.88:6821/3916094166","state":["exists","up"]},{"osd":7,"uuid":"5fdda449-e4d6-4e25-985f-b5b57d4eaed2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6824","nonce":897007524},{"type":"v1","addr":"172.21.15.88:6825","nonce":897007524}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6826","nonce":897007524},{"type":"v1","addr":"172.21.15.88:6827","nonce":897007524}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6830","nonce":897007524},{"type":"v1","addr":"172.21.15.88:6831","nonce":897007524}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6828","nonce":897007524},{"type":"v1","addr":"172.21.15.88:6829","nonce":897007524}]},"public_addr":"172.21.15.88:6825/897007524","cluster_addr":"172.21.15.88:6827/897007524","heartbeat_back_addr":"172.21.15.88:6831/897007524","heartbeat_front_addr":"172.21.15.88:6829/897007524","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T09:30:38.737600+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T09:31:12.091839+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T09:31:39.117143+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T09:32:06.012629+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T09:32:27.785201+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T09:32:51.896300+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T09:33:17.182195+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T09:33:43.138963+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.78:6800/3897938674":"2024-09-17T09:29:28.526174+0000","172.21.15.78:0/1692915456":"2024-09-17T09:29:28.526174+0000","172.21.15.78:6800/2118705212":"2024-09-17T09:27:13.010642+0000","172.21.15.78:6801/3897938674":"2024-09-17T09:29:28.526174+0000","172.21.15.78:0/1394375203":"2024-09-17T09:26:43.343502+0000","172.21.15.78:0/771928015":"2024-09-17T09:29:28.526174+0000","172.21.15.78:0/1889899684":"2024-09-17T09:27:13.010642+0000","172.21.15.78:0/4262597131":"2024-09-17T09:27:13.010642+0000","172.21.15.78:0/2833844034":"2024-09-17T09:26:43.343502+0000","172.21.15.78:0/3399595328":"2024-09-17T09:27:13.010642+0000","172.21.15.78:6801/2118705212":"2024-09-17T09:27:13.010642+0000","172.21.15.78:6800/2084930300":"2024-09-17T09:26:43.343502+0000","172.21.15.78:0/3767531392":"2024-09-17T09:26:43.343502+0000","172.21.15.78:0/1416709728":"2024-09-17T09:29:28.526174+0000","172.21.15.78:6801/2084930300":"2024-09-17T09:26:43.343502+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-16T09:34:00.748 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:00 smithi078 ceph-mon[23694]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:00.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:00 smithi088 ceph-mon[29151]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:01.364 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-16T09:34:01.364 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-16T09:34:01.571 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-16T09:34:01.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:01 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/3184994933' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-16T09:34:01.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:01 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/3184994933' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-16T09:34:02.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:02 smithi078 ceph-mon[23694]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:02.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:02 smithi088 ceph-mon[29151]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:03.202 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-16T09:34:03.202 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":59,"fsid":"a70c765e-740d-11ef-bceb-c7b262605968","created":"2024-09-16T09:26:11.529941+0000","modified":"2024-09-16T09:33:48.465837+0000","last_up_change":"2024-09-16T09:33:44.443556+0000","last_in_change":"2024-09-16T09:33:27.045434+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-16T09:31:41.644068+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"9c561ce9-358c-4e1f-aaa0-497e8410c604","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":3273201876},{"type":"v1","addr":"172.21.15.78:6803","nonce":3273201876}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":3273201876},{"type":"v1","addr":"172.21.15.78:6805","nonce":3273201876}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":3273201876},{"type":"v1","addr":"172.21.15.78:6809","nonce":3273201876}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":3273201876},{"type":"v1","addr":"172.21.15.78:6807","nonce":3273201876}]},"public_addr":"172.21.15.78:6803/3273201876","cluster_addr":"172.21.15.78:6805/3273201876","heartbeat_back_addr":"172.21.15.78:6809/3273201876","heartbeat_front_addr":"172.21.15.78:6807/3273201876","state":["exists","up"]},{"osd":1,"uuid":"0af2d2ef-0835-4383-add0-fc6a751b627f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":3035110387},{"type":"v1","addr":"172.21.15.78:6811","nonce":3035110387}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":3035110387},{"type":"v1","addr":"172.21.15.78:6813","nonce":3035110387}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":3035110387},{"type":"v1","addr":"172.21.15.78:6817","nonce":3035110387}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":3035110387},{"type":"v1","addr":"172.21.15.78:6815","nonce":3035110387}]},"public_addr":"172.21.15.78:6811/3035110387","cluster_addr":"172.21.15.78:6813/3035110387","heartbeat_back_addr":"172.21.15.78:6817/3035110387","heartbeat_front_addr":"172.21.15.78:6815/3035110387","state":["exists","up"]},{"osd":2,"uuid":"08546747-0827-4ff9-b0da-3fbd6d999160","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":150643382},{"type":"v1","addr":"172.21.15.78:6819","nonce":150643382}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":150643382},{"type":"v1","addr":"172.21.15.78:6821","nonce":150643382}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6824","nonce":150643382},{"type":"v1","addr":"172.21.15.78:6825","nonce":150643382}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":150643382},{"type":"v1","addr":"172.21.15.78:6823","nonce":150643382}]},"public_addr":"172.21.15.78:6819/150643382","cluster_addr":"172.21.15.78:6821/150643382","heartbeat_back_addr":"172.21.15.78:6825/150643382","heartbeat_front_addr":"172.21.15.78:6823/150643382","state":["exists","up"]},{"osd":3,"uuid":"17283fa7-427a-4a02-b313-a39a39e9b4e7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6826","nonce":1372470838},{"type":"v1","addr":"172.21.15.78:6827","nonce":1372470838}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6828","nonce":1372470838},{"type":"v1","addr":"172.21.15.78:6829","nonce":1372470838}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6832","nonce":1372470838},{"type":"v1","addr":"172.21.15.78:6833","nonce":1372470838}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6830","nonce":1372470838},{"type":"v1","addr":"172.21.15.78:6831","nonce":1372470838}]},"public_addr":"172.21.15.78:6827/1372470838","cluster_addr":"172.21.15.78:6829/1372470838","heartbeat_back_addr":"172.21.15.78:6833/1372470838","heartbeat_front_addr":"172.21.15.78:6831/1372470838","state":["exists","up"]},{"osd":4,"uuid":"21be2fa4-3506-4335-b86c-397b53bb3cc8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6800","nonce":383049394},{"type":"v1","addr":"172.21.15.88:6801","nonce":383049394}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6802","nonce":383049394},{"type":"v1","addr":"172.21.15.88:6803","nonce":383049394}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6806","nonce":383049394},{"type":"v1","addr":"172.21.15.88:6807","nonce":383049394}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6804","nonce":383049394},{"type":"v1","addr":"172.21.15.88:6805","nonce":383049394}]},"public_addr":"172.21.15.88:6801/383049394","cluster_addr":"172.21.15.88:6803/383049394","heartbeat_back_addr":"172.21.15.88:6807/383049394","heartbeat_front_addr":"172.21.15.88:6805/383049394","state":["exists","up"]},{"osd":5,"uuid":"4016533c-5627-4110-807c-63c97df53ee2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6808","nonce":2909693001},{"type":"v1","addr":"172.21.15.88:6809","nonce":2909693001}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6810","nonce":2909693001},{"type":"v1","addr":"172.21.15.88:6811","nonce":2909693001}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6814","nonce":2909693001},{"type":"v1","addr":"172.21.15.88:6815","nonce":2909693001}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6812","nonce":2909693001},{"type":"v1","addr":"172.21.15.88:6813","nonce":2909693001}]},"public_addr":"172.21.15.88:6809/2909693001","cluster_addr":"172.21.15.88:6811/2909693001","heartbeat_back_addr":"172.21.15.88:6815/2909693001","heartbeat_front_addr":"172.21.15.88:6813/2909693001","state":["exists","up"]},{"osd":6,"uuid":"f9d66824-5a29-441d-ba52-ba79467980c0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6816","nonce":3916094166},{"type":"v1","addr":"172.21.15.88:6817","nonce":3916094166}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6818","nonce":3916094166},{"type":"v1","addr":"172.21.15.88:6819","nonce":3916094166}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6822","nonce":3916094166},{"type":"v1","addr":"172.21.15.88:6823","nonce":3916094166}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6820","nonce":3916094166},{"type":"v1","addr":"172.21.15.88:6821","nonce":3916094166}]},"public_addr":"172.21.15.88:6817/3916094166","cluster_addr":"172.21.15.88:6819/3916094166","heartbeat_back_addr":"172.21.15.88:6823/3916094166","heartbeat_front_addr":"172.21.15.88:6821/3916094166","state":["exists","up"]},{"osd":7,"uuid":"5fdda449-e4d6-4e25-985f-b5b57d4eaed2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6824","nonce":897007524},{"type":"v1","addr":"172.21.15.88:6825","nonce":897007524}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6826","nonce":897007524},{"type":"v1","addr":"172.21.15.88:6827","nonce":897007524}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6830","nonce":897007524},{"type":"v1","addr":"172.21.15.88:6831","nonce":897007524}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6828","nonce":897007524},{"type":"v1","addr":"172.21.15.88:6829","nonce":897007524}]},"public_addr":"172.21.15.88:6825/897007524","cluster_addr":"172.21.15.88:6827/897007524","heartbeat_back_addr":"172.21.15.88:6831/897007524","heartbeat_front_addr":"172.21.15.88:6829/897007524","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T09:30:38.737600+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T09:31:12.091839+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T09:31:39.117143+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T09:32:06.012629+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T09:32:27.785201+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T09:32:51.896300+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T09:33:17.182195+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T09:33:43.138963+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.78:6800/3897938674":"2024-09-17T09:29:28.526174+0000","172.21.15.78:0/1692915456":"2024-09-17T09:29:28.526174+0000","172.21.15.78:6800/2118705212":"2024-09-17T09:27:13.010642+0000","172.21.15.78:6801/3897938674":"2024-09-17T09:29:28.526174+0000","172.21.15.78:0/1394375203":"2024-09-17T09:26:43.343502+0000","172.21.15.78:0/771928015":"2024-09-17T09:29:28.526174+0000","172.21.15.78:0/1889899684":"2024-09-17T09:27:13.010642+0000","172.21.15.78:0/4262597131":"2024-09-17T09:27:13.010642+0000","172.21.15.78:0/2833844034":"2024-09-17T09:26:43.343502+0000","172.21.15.78:0/3399595328":"2024-09-17T09:27:13.010642+0000","172.21.15.78:6801/2118705212":"2024-09-17T09:27:13.010642+0000","172.21.15.78:6800/2084930300":"2024-09-17T09:26:43.343502+0000","172.21.15.78:0/3767531392":"2024-09-17T09:26:43.343502+0000","172.21.15.78:0/1416709728":"2024-09-17T09:29:28.526174+0000","172.21.15.78:6801/2084930300":"2024-09-17T09:26:43.343502+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-16T09:34:03.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:03 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/2973431770' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-16T09:34:03.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:03 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/2973431770' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-16T09:34:03.810 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-16T09:34:03.811 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-16T09:34:03.811 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-16T09:34:03.811 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-16T09:34:03.812 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-16T09:34:03.812 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-16T09:34:03.813 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-16T09:34:03.813 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-16T09:34:04.205 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-16T09:34:04.212 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-16T09:34:04.215 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-16T09:34:04.215 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-16T09:34:04.218 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-16T09:34:04.220 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-16T09:34:04.223 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-16T09:34:04.223 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-16T09:34:04.674 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:04 smithi078 ceph-mon[23694]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:04.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:04 smithi088 ceph-mon[29151]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:06.797 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:06 smithi078 ceph-mon[23694]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:06.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:06 smithi088 ceph-mon[29151]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:08.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:08 smithi088 ceph-mon[29151]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:08.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:08 smithi078 ceph-mon[23694]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:10.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:10 smithi088 ceph-mon[29151]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:10.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:10 smithi078 ceph-mon[23694]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:12.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:12 smithi088 ceph-mon[29151]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:12.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:12 smithi078 ceph-mon[23694]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:13.240 INFO:teuthology.orchestra.run.smithi078.stdout:133143986200 2024-09-16T09:34:13.241 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-16T09:34:14.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:14 smithi078 ceph-mon[23694]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:14.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:14 smithi088 ceph-mon[29151]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:15.431 INFO:teuthology.orchestra.run.smithi078.stdout:236223201289 2024-09-16T09:34:15.431 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-16T09:34:15.991 INFO:teuthology.orchestra.run.smithi078.stdout:167503724563 2024-09-16T09:34:15.992 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-16T09:34:16.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:16 smithi078 ceph-mon[23694]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:16.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:16 smithi088 ceph-mon[29151]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:16.957 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-16T09:34:17.435 INFO:teuthology.orchestra.run.smithi078.stdout:38654705710 2024-09-16T09:34:17.435 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-16T09:34:18.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:18 smithi078 ceph-mon[23694]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:18.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:18 smithi088 ceph-mon[29151]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:18.808 INFO:teuthology.orchestra.run.smithi078.stdout:60129542183 2024-09-16T09:34:18.808 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-16T09:34:19.437 INFO:teuthology.orchestra.run.smithi078.stdout:201863462926 2024-09-16T09:34:19.437 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-16T09:34:20.386 INFO:teuthology.orchestra.run.smithi078.stdout:111669149725 2024-09-16T09:34:20.387 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-16T09:34:20.706 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:20 smithi078 ceph-mon[23694]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:20.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:20 smithi088 ceph-mon[29151]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:21.028 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-16T09:34:21.029 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-16T09:34:21.029 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-16T09:34:21.029 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-16T09:34:21.457 INFO:teuthology.orchestra.run.smithi078.stdout:81604378658 2024-09-16T09:34:21.457 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-16T09:34:22.667 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:22 smithi078 ceph-mon[23694]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:22.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:22 smithi088 ceph-mon[29151]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:23.789 INFO:teuthology.orchestra.run.smithi078.stdout:133143986202 2024-09-16T09:34:24.487 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986200 got 133143986202 for osd.4 2024-09-16T09:34:24.487 DEBUG:teuthology.parallel:result is None 2024-09-16T09:34:24.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:24 smithi088 ceph-mon[29151]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:24.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:24 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/3965307717' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-16T09:34:24.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:24 smithi078 ceph-mon[23694]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:24.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:24 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/3965307717' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-16T09:34:24.929 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-16T09:34:24.930 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-16T09:34:24.930 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-16T09:34:26.710 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:26 smithi078 ceph-mon[23694]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:26.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:26 smithi088 ceph-mon[29151]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:28.553 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:28 smithi078 ceph-mon[23694]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:28.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:28 smithi088 ceph-mon[29151]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:29 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:34:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:29 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/trash_purge_schedule"}]: dispatch 2024-09-16T09:34:29.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:29 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:34:29.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:29 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/trash_purge_schedule"}]: dispatch 2024-09-16T09:34:30.652 INFO:teuthology.orchestra.run.smithi078.stdout:38654705713 2024-09-16T09:34:30.743 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:30 smithi078 ceph-mon[23694]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:30.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:30 smithi088 ceph-mon[29151]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:30.843 INFO:teuthology.orchestra.run.smithi078.stdout:236223201292 2024-09-16T09:34:31.491 INFO:teuthology.orchestra.run.smithi078.stdout:167503724566 2024-09-16T09:34:31.775 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:31 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/3673845540' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-16T09:34:31.776 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:31 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/311652221' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-16T09:34:31.776 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:31 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/2032972627' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-16T09:34:32.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:31 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/3673845540' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-16T09:34:32.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:31 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/311652221' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-16T09:34:32.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:31 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/2032972627' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-16T09:34:32.177 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705710 got 38654705713 for osd.0 2024-09-16T09:34:32.178 DEBUG:teuthology.parallel:result is None 2024-09-16T09:34:32.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:32 smithi078 ceph-mon[23694]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:33.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:32 smithi088 ceph-mon[29151]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:33.190 INFO:teuthology.orchestra.run.smithi078.stdout:60129542187 2024-09-16T09:34:33.391 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201289 got 236223201292 for osd.7 2024-09-16T09:34:33.391 DEBUG:teuthology.parallel:result is None 2024-09-16T09:34:33.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:33 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/781680527' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-16T09:34:33.975 INFO:teuthology.orchestra.run.smithi078.stdout:81604378661 2024-09-16T09:34:34.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:33 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/781680527' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-16T09:34:34.398 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724563 got 167503724566 for osd.5 2024-09-16T09:34:34.398 DEBUG:teuthology.parallel:result is None 2024-09-16T09:34:34.714 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:34 smithi078 ceph-mon[23694]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:34.714 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:34 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/3599410478' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-16T09:34:34.993 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378658 got 81604378661 for osd.2 2024-09-16T09:34:34.993 DEBUG:teuthology.parallel:result is None 2024-09-16T09:34:35.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:34 smithi088 ceph-mon[29151]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:35.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:34 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/3599410478' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-16T09:34:35.472 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542183 got 60129542187 for osd.1 2024-09-16T09:34:35.472 DEBUG:teuthology.parallel:result is None 2024-09-16T09:34:36.313 INFO:teuthology.orchestra.run.smithi078.stdout:111669149729 2024-09-16T09:34:36.504 INFO:teuthology.orchestra.run.smithi078.stdout:201863462930 2024-09-16T09:34:36.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:36 smithi078 ceph-mon[23694]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:36.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:36 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/2029505512' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-16T09:34:36.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:36 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/3689995461' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-16T09:34:37.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:36 smithi088 ceph-mon[29151]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:37.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:36 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/2029505512' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-16T09:34:37.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:36 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/3689995461' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-16T09:34:37.119 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462926 got 201863462930 for osd.6 2024-09-16T09:34:37.119 DEBUG:teuthology.parallel:result is None 2024-09-16T09:34:37.583 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149725 got 111669149729 for osd.3 2024-09-16T09:34:37.584 DEBUG:teuthology.parallel:result is None 2024-09-16T09:34:37.584 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-16T09:34:37.584 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-16T09:34:37.797 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-16T09:34:37.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:37 smithi078 ceph-mon[23694]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:38.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:37 smithi088 ceph-mon[29151]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:39.453 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-16T09:34:39.455 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2024-09-16T09:34:40.098 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":202,"stamp":"2024-09-16T09:34:39.390196+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48860,"kb_used_data":6300,"kb_used_omap":0,"kb_used_meta":42496,"kb_avail":749912356,"statfs":{"total":767960285184,"available":767910252544,"internally_reserved":0,"allocated":6451200,"data_stored":3546281,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43515904},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002130"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-16T09:33:48.475984+0000","last_change":"2024-09-16T09:33:48.475984+0000","last_active":"2024-09-16T09:33:48.475984+0000","last_peered":"2024-09-16T09:33:48.475984+0000","last_clean":"2024-09-16T09:33:48.475984+0000","last_became_active":"2024-09-16T09:33:48.475506+0000","last_became_peered":"2024-09-16T09:33:48.475506+0000","last_unstale":"2024-09-16T09:33:48.475984+0000","last_undegraded":"2024-09-16T09:33:48.475984+0000","last_fullsized":"2024-09-16T09:33:48.475984+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-16T09:31:41.644484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-16T09:31:41.644484+0000","last_clean_scrub_stamp":"2024-09-16T09:31:41.644484+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-17T15:47:17.021055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3194880,"data_stored":3173568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201294,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6048,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739104,"statfs":{"total":95995035648,"available":95988842496,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67400000000000004}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64500000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65600000000000003}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53900000000000003}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45100000000000001}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58799999999999997}]}]},{"osd":6,"up_from":47,"seq":201863462931,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6180,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738972,"statfs":{"total":95995035648,"available":95988707328,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 09:34:20 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.53700000000000003}]},{"osd":1,"last update":"Mon Sep 16 09:34:20 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.56399999999999995}]},{"osd":2,"last update":"Mon Sep 16 09:34:20 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.433}]},{"osd":3,"last update":"Mon Sep 16 09:34:20 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.35999999999999999}]},{"osd":4,"last update":"Mon Sep 16 09:34:20 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.501}]},{"osd":5,"last update":"Mon Sep 16 09:34:20 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.51100000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47999999999999998}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6356,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738796,"statfs":{"total":95995035648,"available":95988527104,"internally_reserved":0,"allocated":602112,"data_stored":244703,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 09:34:17 2024","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.33200000000000002,"15min":0.308},"min":{"1min":0.23599999999999999,"5min":0.216,"15min":0.216},"max":{"1min":0.58099999999999996,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.35099999999999998,"15min":0.32700000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.52600000000000002}]},{"osd":2,"last update":"Mon Sep 16 09:33:47 2024","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.378,"15min":0.37},"min":{"1min":0.28499999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.56799999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.439,"15min":0.433},"min":{"1min":0.33500000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.55500000000000005}]},{"osd":3,"last update":"Mon Sep 16 09:34:11 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.46600000000000003,"15min":0.46999999999999997},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.438,"15min":0.44},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.56299999999999994}]},{"osd":4,"last update":"Mon Sep 16 09:33:33 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.625},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.753}]},{"osd":5,"last update":"Mon Sep 16 09:33:57 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.81599999999999995},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.76100000000000001}]},{"osd":6,"last update":"Mon Sep 16 09:34:19 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.78400000000000003},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.85999999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.79900000000000004}]}]},{"osd":0,"up_from":9,"seq":38654705715,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Sep 16 09:34:20 2024","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.317,"15min":0.29999999999999999},"min":{"1min":0.23699999999999999,"5min":0.215,"15min":0.215},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.33100000000000002,"15min":0.307},"min":{"1min":0.252,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.32200000000000001}]},{"osd":2,"last update":"Mon Sep 16 09:33:47 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.36599999999999999,"15min":0.35999999999999999},"min":{"1min":0.308,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.39400000000000002,"15min":0.38400000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.38100000000000001}]},{"osd":3,"last update":"Mon Sep 16 09:34:09 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.42899999999999999,"15min":0.42699999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.438},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.45800000000000002,"15min":0.45600000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.47499999999999998}]},{"osd":4,"last update":"Mon Sep 16 09:34:34 2024","interfaces":[{"interface":"back","average":{"1min":1.3340000000000001,"5min":0.68899999999999995,"15min":0.58099999999999996},"min":{"1min":0.434,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":13.444000000000001,"5min":13.444000000000001,"15min":13.444000000000001},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.55400000000000005,"15min":0.54400000000000004},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.46999999999999997}]},{"osd":5,"last update":"Mon Sep 16 09:33:55 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.498}]},{"osd":6,"last update":"Mon Sep 16 09:34:18 2024","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":1.2569999999999999,"15min":1.2569999999999999},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":13.407,"5min":13.407,"15min":13.407},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.51000000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55400000000000005}]}]},{"osd":2,"up_from":19,"seq":81604378662,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6232,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738920,"statfs":{"total":95995035648,"available":95988654080,"internally_reserved":0,"allocated":606208,"data_stored":244572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 09:33:42 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.379,"15min":0.36799999999999999},"min":{"1min":0.28499999999999998,"5min":0.219,"15min":0.219},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.375,"15min":0.36599999999999999},"min":{"1min":0.24199999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.49399999999999999}]},{"osd":1,"last update":"Mon Sep 16 09:33:42 2024","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.39500000000000002,"15min":0.38900000000000001},"min":{"1min":0.254,"5min":0.251,"15min":0.251},"max":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.39300000000000002,"15min":0.38100000000000001},"min":{"1min":0.32200000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.40999999999999998}]},{"osd":3,"last update":"Mon Sep 16 09:34:16 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.42499999999999999,"15min":0.42699999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.57199999999999995,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.62,"5min":0.62,"15min":0.62},"last":0.51200000000000001}]},{"osd":4,"last update":"Mon Sep 16 09:34:30 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.56799999999999995,"15min":0.56200000000000006},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.82999999999999996},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.56799999999999995,"15min":0.56399999999999995},"min":{"1min":0.39400000000000002,"5min":0.376,"15min":0.376},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.76900000000000002}]},{"osd":5,"last update":"Mon Sep 16 09:33:56 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.745},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.80200000000000005}]},{"osd":6,"last update":"Mon Sep 16 09:34:20 2024","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.81999999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.81200000000000006}]}]},{"osd":3,"up_from":26,"seq":111669149730,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 09:34:13 2024","interfaces":[{"interface":"back","average":{"1min":0.33700000000000002,"5min":0.36499999999999999,"15min":0.36899999999999999},"min":{"1min":0.182,"5min":0.182,"15min":0.182},"max":{"1min":0.45800000000000002,"5min":0.55100000000000005,"15min":0.55100000000000005},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.38600000000000001,"15min":0.39100000000000001},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.433}]},{"osd":1,"last update":"Mon Sep 16 09:34:13 2024","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.39700000000000002,"15min":0.40100000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.51200000000000001,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.379,"5min":0.39600000000000002,"15min":0.39900000000000002},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.57499999999999996,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.45100000000000001}]},{"osd":2,"last update":"Mon Sep 16 09:34:13 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.436,"15min":0.436},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.64500000000000002,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.435,"15min":0.438},"min":{"1min":0.26700000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.40999999999999998}]},{"osd":4,"last update":"Mon Sep 16 09:34:33 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56100000000000005,"15min":0.56000000000000005},"min":{"1min":0.377,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.76200000000000001,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.58099999999999996,"15min":0.57699999999999996},"min":{"1min":0.42299999999999999,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.61199999999999999}]},{"osd":5,"last update":"Mon Sep 16 09:33:54 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.68000000000000005}]},{"osd":6,"last update":"Mon Sep 16 09:34:20 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.65300000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60599999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986205,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5788,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739364,"statfs":{"total":95995035648,"available":95989108736,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 09:34:34 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.54100000000000004,"15min":0.54500000000000004},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.70799999999999996,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.53500000000000003,"15min":0.53300000000000003},"min":{"1min":0.41499999999999998,"5min":0.371,"15min":0.371},"max":{"1min":0.69099999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.54100000000000004}]},{"osd":1,"last update":"Mon Sep 16 09:34:34 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.52100000000000002,"15min":0.52500000000000002},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.629,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.52500000000000002,"15min":0.52100000000000002},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.56799999999999995}]},{"osd":2,"last update":"Mon Sep 16 09:34:34 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.41999999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.72299999999999998,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.55600000000000005,"15min":0.55800000000000005},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.58199999999999996}]},{"osd":3,"last update":"Mon Sep 16 09:34:34 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.54500000000000004,"15min":0.53800000000000003},"min":{"1min":0.42699999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.745,"5min":0.751,"15min":0.751},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.55500000000000005,"15min":0.55800000000000005},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.51500000000000001}]},{"osd":5,"last update":"Mon Sep 16 09:33:54 2024","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.442},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.496}]},{"osd":6,"last update":"Mon Sep 16 09:34:19 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.60899999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67100000000000004}]}]},{"osd":5,"up_from":39,"seq":167503724568,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5652,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739500,"statfs":{"total":95995035648,"available":95989248000,"internally_reserved":0,"allocated":602112,"data_stored":244645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 09:33:54 2024","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.3679999999999999,"5min":2.3679999999999999,"15min":2.3679999999999999},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":2.3860000000000001,"5min":2.3860000000000001,"15min":2.3860000000000001},"last":0.66700000000000004}]},{"osd":1,"last update":"Mon Sep 16 09:33:54 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.423,"5min":2.423,"15min":2.423},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":2.403,"5min":2.403,"15min":2.403},"last":0.64100000000000001}]},{"osd":2,"last update":"Mon Sep 16 09:33:54 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.3239999999999998,"5min":2.3239999999999998,"15min":2.3239999999999998},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":2.3799999999999999,"5min":2.3799999999999999,"15min":2.3799999999999999},"last":0.65200000000000002}]},{"osd":3,"last update":"Mon Sep 16 09:33:54 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.3599999999999999,"5min":2.3599999999999999,"15min":2.3599999999999999},"last":0.626},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":2.3330000000000002,"5min":2.3330000000000002,"15min":2.3330000000000002},"last":0.68700000000000006}]},{"osd":4,"last update":"Mon Sep 16 09:33:54 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.497},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.53700000000000003}]},{"osd":6,"last update":"Mon Sep 16 09:34:19 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.69599999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73499999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-16T09:34:40.100 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-16T09:34:40.310 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-16T09:34:40.667 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:40 smithi078 ceph-mon[23694]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:40.667 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:40 smithi078 ceph-mon[23694]: from='client.14498 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T09:34:40.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:40 smithi088 ceph-mon[29151]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:40.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:40 smithi088 ceph-mon[29151]: from='client.14498 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T09:34:42.092 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-16T09:34:42.093 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2024-09-16T09:34:42.678 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:42 smithi078 ceph-mon[23694]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:42.701 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":203,"stamp":"2024-09-16T09:34:41.390546+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48860,"kb_used_data":6300,"kb_used_omap":0,"kb_used_meta":42496,"kb_avail":749912356,"statfs":{"total":767960285184,"available":767910252544,"internally_reserved":0,"allocated":6451200,"data_stored":3546281,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43515904},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002030"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-16T09:33:48.475984+0000","last_change":"2024-09-16T09:33:48.475984+0000","last_active":"2024-09-16T09:33:48.475984+0000","last_peered":"2024-09-16T09:33:48.475984+0000","last_clean":"2024-09-16T09:33:48.475984+0000","last_became_active":"2024-09-16T09:33:48.475506+0000","last_became_peered":"2024-09-16T09:33:48.475506+0000","last_unstale":"2024-09-16T09:33:48.475984+0000","last_undegraded":"2024-09-16T09:33:48.475984+0000","last_fullsized":"2024-09-16T09:33:48.475984+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-16T09:31:41.644484+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-16T09:31:41.644484+0000","last_clean_scrub_stamp":"2024-09-16T09:31:41.644484+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-17T15:47:17.021055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3194880,"data_stored":3173568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201295,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6048,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739104,"statfs":{"total":95995035648,"available":95988842496,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41599999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.495}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51600000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45500000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59299999999999997}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65200000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64100000000000001}]}]},{"osd":6,"up_from":47,"seq":201863462931,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6180,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738972,"statfs":{"total":95995035648,"available":95988707328,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 09:34:20 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.53700000000000003}]},{"osd":1,"last update":"Mon Sep 16 09:34:20 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.56399999999999995}]},{"osd":2,"last update":"Mon Sep 16 09:34:20 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.433}]},{"osd":3,"last update":"Mon Sep 16 09:34:20 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.35999999999999999}]},{"osd":4,"last update":"Mon Sep 16 09:34:20 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.501}]},{"osd":5,"last update":"Mon Sep 16 09:34:20 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.51100000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47999999999999998}]}]},{"osd":1,"up_from":14,"seq":60129542189,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6356,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738796,"statfs":{"total":95995035648,"available":95988527104,"internally_reserved":0,"allocated":602112,"data_stored":244703,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 09:34:17 2024","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.33200000000000002,"15min":0.308},"min":{"1min":0.23599999999999999,"5min":0.216,"15min":0.216},"max":{"1min":0.58099999999999996,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.35099999999999998,"15min":0.32700000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.40699999999999997}]},{"osd":2,"last update":"Mon Sep 16 09:33:47 2024","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.378,"15min":0.37},"min":{"1min":0.28499999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.56799999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.439,"15min":0.433},"min":{"1min":0.33500000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.34200000000000003}]},{"osd":3,"last update":"Mon Sep 16 09:34:11 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.46600000000000003,"15min":0.46999999999999997},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.38},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.438,"15min":0.44},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.42099999999999999}]},{"osd":4,"last update":"Mon Sep 16 09:34:35 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.57999999999999996,"15min":0.57599999999999996},"min":{"1min":0.39700000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.58599999999999997,"15min":0.57499999999999996},"min":{"1min":0.46300000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.59499999999999997}]},{"osd":5,"last update":"Mon Sep 16 09:33:57 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.50900000000000001}]},{"osd":6,"last update":"Mon Sep 16 09:34:19 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.46800000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68999999999999995}]}]},{"osd":0,"up_from":9,"seq":38654705715,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Sep 16 09:34:20 2024","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.317,"15min":0.29999999999999999},"min":{"1min":0.23699999999999999,"5min":0.215,"15min":0.215},"max":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.33100000000000002,"15min":0.307},"min":{"1min":0.252,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.32200000000000001}]},{"osd":2,"last update":"Mon Sep 16 09:33:47 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.36599999999999999,"15min":0.35999999999999999},"min":{"1min":0.308,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.39400000000000002,"15min":0.38400000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.38100000000000001}]},{"osd":3,"last update":"Mon Sep 16 09:34:09 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.42899999999999999,"15min":0.42699999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.438},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.45800000000000002,"15min":0.45600000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.47499999999999998}]},{"osd":4,"last update":"Mon Sep 16 09:34:34 2024","interfaces":[{"interface":"back","average":{"1min":1.3340000000000001,"5min":0.68899999999999995,"15min":0.58099999999999996},"min":{"1min":0.434,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":13.444000000000001,"5min":13.444000000000001,"15min":13.444000000000001},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.55400000000000005,"15min":0.54400000000000004},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.46999999999999997}]},{"osd":5,"last update":"Mon Sep 16 09:33:55 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.498}]},{"osd":6,"last update":"Mon Sep 16 09:34:18 2024","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":1.2569999999999999,"15min":1.2569999999999999},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":13.407,"5min":13.407,"15min":13.407},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.51000000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55400000000000005}]}]},{"osd":2,"up_from":19,"seq":81604378662,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6232,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738920,"statfs":{"total":95995035648,"available":95988654080,"internally_reserved":0,"allocated":606208,"data_stored":244572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 09:33:42 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.379,"15min":0.36799999999999999},"min":{"1min":0.28499999999999998,"5min":0.219,"15min":0.219},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.375,"15min":0.36599999999999999},"min":{"1min":0.24199999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.49399999999999999}]},{"osd":1,"last update":"Mon Sep 16 09:33:42 2024","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.39500000000000002,"15min":0.38900000000000001},"min":{"1min":0.254,"5min":0.251,"15min":0.251},"max":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.39300000000000002,"15min":0.38100000000000001},"min":{"1min":0.32200000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.40999999999999998}]},{"osd":3,"last update":"Mon Sep 16 09:34:16 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.42499999999999999,"15min":0.42699999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.57199999999999995,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.62,"5min":0.62,"15min":0.62},"last":0.51200000000000001}]},{"osd":4,"last update":"Mon Sep 16 09:34:30 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.56799999999999995,"15min":0.56200000000000006},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.82999999999999996},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.56799999999999995,"15min":0.56399999999999995},"min":{"1min":0.39400000000000002,"5min":0.376,"15min":0.376},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.76900000000000002}]},{"osd":5,"last update":"Mon Sep 16 09:33:56 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.745},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.80200000000000005}]},{"osd":6,"last update":"Mon Sep 16 09:34:20 2024","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.81999999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.81200000000000006}]}]},{"osd":3,"up_from":26,"seq":111669149730,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 09:34:13 2024","interfaces":[{"interface":"back","average":{"1min":0.33700000000000002,"5min":0.36499999999999999,"15min":0.36899999999999999},"min":{"1min":0.182,"5min":0.182,"15min":0.182},"max":{"1min":0.45800000000000002,"5min":0.55100000000000005,"15min":0.55100000000000005},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.35399999999999998,"5min":0.38600000000000001,"15min":0.39100000000000001},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.433}]},{"osd":1,"last update":"Mon Sep 16 09:34:13 2024","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.39700000000000002,"15min":0.40100000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.51200000000000001,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.379,"5min":0.39600000000000002,"15min":0.39900000000000002},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.57499999999999996,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.45100000000000001}]},{"osd":2,"last update":"Mon Sep 16 09:34:13 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.436,"15min":0.436},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.64500000000000002,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.435,"15min":0.438},"min":{"1min":0.26700000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.40999999999999998}]},{"osd":4,"last update":"Mon Sep 16 09:34:33 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56100000000000005,"15min":0.56000000000000005},"min":{"1min":0.377,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.76200000000000001,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.58099999999999996,"15min":0.57699999999999996},"min":{"1min":0.42299999999999999,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.61199999999999999}]},{"osd":5,"last update":"Mon Sep 16 09:33:54 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.68000000000000005}]},{"osd":6,"last update":"Mon Sep 16 09:34:20 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.65300000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60599999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986206,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5788,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739364,"statfs":{"total":95995035648,"available":95989108736,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 09:34:34 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.54100000000000004,"15min":0.54500000000000004},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.70799999999999996,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.53500000000000003,"15min":0.53300000000000003},"min":{"1min":0.41499999999999998,"5min":0.371,"15min":0.371},"max":{"1min":0.69099999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.59899999999999998}]},{"osd":1,"last update":"Mon Sep 16 09:34:34 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.52100000000000002,"15min":0.52500000000000002},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.629,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.52500000000000002,"15min":0.52100000000000002},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.52700000000000002}]},{"osd":2,"last update":"Mon Sep 16 09:34:34 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.41999999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.72299999999999998,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.55600000000000005,"15min":0.55800000000000005},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.56200000000000006}]},{"osd":3,"last update":"Mon Sep 16 09:34:34 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.54500000000000004,"15min":0.53800000000000003},"min":{"1min":0.42699999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.745,"5min":0.751,"15min":0.751},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.55500000000000005,"15min":0.55800000000000005},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.58499999999999996}]},{"osd":5,"last update":"Mon Sep 16 09:33:54 2024","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.64500000000000002}]},{"osd":6,"last update":"Mon Sep 16 09:34:19 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.63}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70899999999999996}]}]},{"osd":5,"up_from":39,"seq":167503724568,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5652,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739500,"statfs":{"total":95995035648,"available":95989248000,"internally_reserved":0,"allocated":602112,"data_stored":244645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 09:33:54 2024","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.3679999999999999,"5min":2.3679999999999999,"15min":2.3679999999999999},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":2.3860000000000001,"5min":2.3860000000000001,"15min":2.3860000000000001},"last":0.66700000000000004}]},{"osd":1,"last update":"Mon Sep 16 09:33:54 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.423,"5min":2.423,"15min":2.423},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":2.403,"5min":2.403,"15min":2.403},"last":0.64100000000000001}]},{"osd":2,"last update":"Mon Sep 16 09:33:54 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.3239999999999998,"5min":2.3239999999999998,"15min":2.3239999999999998},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":2.3799999999999999,"5min":2.3799999999999999,"15min":2.3799999999999999},"last":0.65200000000000002}]},{"osd":3,"last update":"Mon Sep 16 09:33:54 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":2.3599999999999999,"5min":2.3599999999999999,"15min":2.3599999999999999},"last":0.626},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":2.3330000000000002,"5min":2.3330000000000002,"15min":2.3330000000000002},"last":0.68700000000000006}]},{"osd":4,"last update":"Mon Sep 16 09:33:54 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.497},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.53700000000000003}]},{"osd":6,"last update":"Mon Sep 16 09:34:19 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.69599999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73499999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-16T09:34:42.703 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-16T09:34:42.703 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-16T09:34:42.703 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-16T09:34:42.703 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-16T09:34:42.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:42 smithi088 ceph-mon[29151]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:42.916 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/mon.smithi078/config 2024-09-16T09:34:43.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:43 smithi078 ceph-mon[23694]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T09:34:43.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:43 smithi088 ceph-mon[29151]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T09:34:44.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:44 smithi078 ceph-mon[23694]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:44.639 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-16T09:34:44.639 INFO:teuthology.orchestra.run.smithi078.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-16T09:34:44.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:44 smithi088 ceph-mon[29151]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:45.317 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-16T09:34:45.318 INFO:tasks.cephadm:Setup complete, yielding 2024-09-16T09:34:45.318 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-16T09:34:45.328 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-16T09:34:45.329 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- bash -c 'ceph orch status' 2024-09-16T09:34:45.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:45 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/4090186815' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-16T09:34:45.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:45 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/4090186815' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-16T09:34:46.580 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:46 smithi078 ceph-mon[23694]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:46.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:46 smithi088 ceph-mon[29151]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:47.292 INFO:teuthology.orchestra.run.smithi078.stdout:Backend: cephadm 2024-09-16T09:34:47.292 INFO:teuthology.orchestra.run.smithi078.stdout:Available: Yes 2024-09-16T09:34:47.292 INFO:teuthology.orchestra.run.smithi078.stdout:Paused: No 2024-09-16T09:34:47.550 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:47 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:34:47.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:47 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:34:47.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:47 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:34:47.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:47 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:34:47.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:47 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:34:47.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:47 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:34:47.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:47 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:34:47.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:47 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:34:47.903 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- bash -c 'ceph orch ps' 2024-09-16T09:34:48.531 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:48 smithi078 ceph-mon[23694]: from='client.14510 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:34:48.532 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:48 smithi078 ceph-mon[23694]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:48.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:48 smithi088 ceph-mon[29151]: from='client.14510 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:34:48.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:48 smithi088 ceph-mon[29151]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:49.955 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T09:34:49.955 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (4m) 2m ago 7m 15.4M - ba2b418f427c 08288e8d0451 2024-09-16T09:34:49.955 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (7m) 2m ago 7m 6405k - 17.2.0 e1d6a67b021e a7184b35ee05 2024-09-16T09:34:49.955 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi088 smithi088 running (5m) 66s ago 5m 6409k - 17.2.0 e1d6a67b021e a9a129cd5d7d 2024-09-16T09:34:49.955 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (4m) 2m ago 5m 46.8M - 8.3.5 dad864ee21e9 53d95a875f02 2024-09-16T09:34:49.955 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rrfxge smithi078 *:9283 running (8m) 2m ago 8m 447M - 17.2.0 e1d6a67b021e 3445d8bfa64d 2024-09-16T09:34:49.955 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi088.ncxfdz smithi088 *:8443,9283 running (5m) 66s ago 5m 414M - 17.2.0 e1d6a67b021e 5971b96856e6 2024-09-16T09:34:49.955 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (8m) 2m ago 8m 45.4M 2048M 17.2.0 e1d6a67b021e c6b35fd1a57a 2024-09-16T09:34:49.955 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi088 smithi088 running (5m) 66s ago 5m 38.1M 2048M 17.2.0 e1d6a67b021e 774a14c00113 2024-09-16T09:34:49.955 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (5m) 2m ago 5m 17.4M - 1dbe0e931976 3716a43984ae 2024-09-16T09:34:49.956 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi088 smithi088 *:9100 running (4m) 66s ago 4m 17.1M - 1dbe0e931976 bc997e95eb63 2024-09-16T09:34:49.956 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (4m) 2m ago 4m 35.4M 3216M 17.2.0 e1d6a67b021e 453389fafdff 2024-09-16T09:34:49.956 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (3m) 2m ago 3m 34.8M 3216M 17.2.0 e1d6a67b021e d8b72a22a11b 2024-09-16T09:34:49.956 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (3m) 2m ago 3m 31.1M 3216M 17.2.0 e1d6a67b021e aa521853c9bf 2024-09-16T09:34:49.956 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (2m) 2m ago 2m 12.0M 3216M 17.2.0 e1d6a67b021e db62172fe90c 2024-09-16T09:34:49.956 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi088 running (2m) 66s ago 2m 33.5M 3984M 17.2.0 e1d6a67b021e 6f4e80fe127a 2024-09-16T09:34:49.956 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi088 running (2m) 66s ago 2m 35.8M 3984M 17.2.0 e1d6a67b021e 73e51edb7756 2024-09-16T09:34:49.956 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi088 running (97s) 66s ago 96s 31.1M 3984M 17.2.0 e1d6a67b021e 7c1459b1a2d9 2024-09-16T09:34:49.956 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi088 running (71s) 66s ago 70s 10.7M 3984M 17.2.0 e1d6a67b021e 37f619c2ddf9 2024-09-16T09:34:49.956 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (4m) 2m ago 4m 52.2M - 514e6a882f6e db9476c8a1d8 2024-09-16T09:34:50.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:50 smithi078 ceph-mon[23694]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:50.626 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- bash -c 'ceph orch ls' 2024-09-16T09:34:50.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:50 smithi088 ceph-mon[29151]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:51.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:51 smithi078 ceph-mon[23694]: from='client.14514 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:34:51.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:51 smithi088 ceph-mon[29151]: from='client.14514 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:34:52.507 INFO:teuthology.orchestra.run.smithi078.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-09-16T09:34:52.508 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager ?:9093,9094 1/1 2m ago 7m count:1 2024-09-16T09:34:52.508 INFO:teuthology.orchestra.run.smithi078.stdout:crash 2/2 2m ago 7m * 2024-09-16T09:34:52.508 INFO:teuthology.orchestra.run.smithi078.stdout:grafana ?:3000 1/1 2m ago 7m count:1 2024-09-16T09:34:52.508 INFO:teuthology.orchestra.run.smithi078.stdout:mgr 2/2 2m ago 7m count:2 2024-09-16T09:34:52.508 INFO:teuthology.orchestra.run.smithi078.stdout:mon 2/2 2m ago 7m smithi078:172.21.15.78=smithi078;smithi088:172.21.15.88=smithi088;count:2 2024-09-16T09:34:52.508 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter ?:9100 2/2 2m ago 7m * 2024-09-16T09:34:52.508 INFO:teuthology.orchestra.run.smithi078.stdout:osd 8 2m ago - 2024-09-16T09:34:52.508 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus ?:9095 1/1 2m ago 7m count:1 2024-09-16T09:34:52.702 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:52 smithi088 ceph-mon[29151]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:52.805 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:52 smithi078 ceph-mon[23694]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:53.106 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- bash -c 'ceph orch host ls' 2024-09-16T09:34:53.742 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:53 smithi078 ceph-mon[23694]: from='client.14518 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:34:53.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:53 smithi088 ceph-mon[29151]: from='client.14518 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:34:54.632 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:54 smithi078 ceph-mon[23694]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:54.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:54 smithi088 ceph-mon[29151]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:55.034 INFO:teuthology.orchestra.run.smithi078.stdout:HOST ADDR LABELS STATUS 2024-09-16T09:34:55.035 INFO:teuthology.orchestra.run.smithi078.stdout:smithi078 172.21.15.78 2024-09-16T09:34:55.035 INFO:teuthology.orchestra.run.smithi078.stdout:smithi088 172.21.15.88 2024-09-16T09:34:55.035 INFO:teuthology.orchestra.run.smithi078.stdout:2 hosts in cluster 2024-09-16T09:34:55.667 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- bash -c 'ceph orch device ls' 2024-09-16T09:34:56.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:56 smithi078 ceph-mon[23694]: from='client.14522 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:34:56.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:56 smithi078 ceph-mon[23694]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:56.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:56 smithi088 ceph-mon[29151]: from='client.14522 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:34:56.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:56 smithi088 ceph-mon[29151]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:57.586 INFO:teuthology.orchestra.run.smithi078.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-09-16T09:34:57.586 INFO:teuthology.orchestra.run.smithi078.stdout:smithi078 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620400QR400BGN 400G LVM detected, locked 2024-09-16T09:34:57.586 INFO:teuthology.orchestra.run.smithi078.stdout:smithi088 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620400XD400BGN 400G LVM detected, locked 2024-09-16T09:34:58.215 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-16T09:34:58.227 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-16T09:34:58.228 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-16T09:34:58.255 INFO:teuthology.orchestra.run.smithi078.stderr:+ systemctl stop nfs-server 2024-09-16T09:34:58.263 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi088.front.sepia.ceph.com 2024-09-16T09:34:58.263 DEBUG:teuthology.orchestra.run.smithi088:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-16T09:34:58.294 INFO:teuthology.orchestra.run.smithi088.stderr:+ systemctl stop nfs-server 2024-09-16T09:34:58.304 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-16T09:34:58.313 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-16T09:34:58.313 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- bash -c 'ceph fs volume create foofs' 2024-09-16T09:34:58.550 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:58 smithi088 ceph-mon[29151]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:58.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:58 smithi078 ceph-mon[23694]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:34:59.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:34:59 smithi078 ceph-mon[23694]: from='client.14526 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:34:59.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:34:59 smithi088 ceph-mon[29151]: from='client.14526 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:35:00.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:00 smithi088 ceph-mon[29151]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:35:01.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:00 smithi078 ceph-mon[23694]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:35:01.961 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:01 smithi078 ceph-mon[23694]: from='client.14530 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:35:01.961 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:01 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-16T09:35:01.961 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:01 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-16T09:35:01.961 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:01 smithi078 ceph-mon[23694]: osdmap e60: 8 total, 8 up, 8 in 2024-09-16T09:35:01.961 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:01 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-16T09:35:01.961 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:01 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi078[23690]: 2024-09-16T09:35:01.621+0000 7f4604a55700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-16T09:35:02.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:01 smithi088 ceph-mon[29151]: from='client.14530 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:35:02.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:01 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-16T09:35:02.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:01 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-16T09:35:02.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:01 smithi088 ceph-mon[29151]: osdmap e60: 8 total, 8 up, 8 in 2024-09-16T09:35:02.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:01 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-16T09:35:02.376 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-09-16T09:35:02.385 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-09-16T09:35:02.385 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-16T09:35:02.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:02 smithi078 ceph-mon[23694]: pgmap v214: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:35:02.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:02 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-16T09:35:02.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:02 smithi078 ceph-mon[23694]: osdmap e61: 8 total, 8 up, 8 in 2024-09-16T09:35:02.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:02 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-16T09:35:02.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:02 smithi078 ceph-mon[23694]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-16T09:35:02.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:02 smithi078 ceph-mon[23694]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-16T09:35:02.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:02 smithi078 ceph-mon[23694]: osdmap e62: 8 total, 8 up, 8 in 2024-09-16T09:35:02.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:02 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-16T09:35:02.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:02 smithi078 ceph-mon[23694]: fsmap foofs:0 2024-09-16T09:35:02.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:02 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:02.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:02 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:35:02.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:02 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:35:02.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:02 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:35:02.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:02 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:02.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:02 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:02.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:02 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.hiotnn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-16T09:35:02.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:02 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.hiotnn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-16T09:35:02.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:02 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:35:03.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:02 smithi088 ceph-mon[29151]: pgmap v214: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:35:03.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:02 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-16T09:35:03.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:02 smithi088 ceph-mon[29151]: osdmap e61: 8 total, 8 up, 8 in 2024-09-16T09:35:03.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:02 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-16T09:35:03.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:02 smithi088 ceph-mon[29151]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-16T09:35:03.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:02 smithi088 ceph-mon[29151]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-16T09:35:03.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:02 smithi088 ceph-mon[29151]: osdmap e62: 8 total, 8 up, 8 in 2024-09-16T09:35:03.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:02 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-16T09:35:03.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:02 smithi088 ceph-mon[29151]: fsmap foofs:0 2024-09-16T09:35:03.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:02 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:03.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:02 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:35:03.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:02 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:35:03.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:02 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:35:03.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:02 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:03.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:02 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:03.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:02 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.hiotnn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-16T09:35:03.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:02 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.hiotnn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-16T09:35:03.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:02 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:35:03.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:03 smithi078 ceph-mon[23694]: Saving service mds.foofs spec with placement count:2 2024-09-16T09:35:03.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:03 smithi078 ceph-mon[23694]: Deploying daemon mds.foofs.smithi078.hiotnn on smithi078 2024-09-16T09:35:03.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:03 smithi078 ceph-mon[23694]: osdmap e63: 8 total, 8 up, 8 in 2024-09-16T09:35:04.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:03 smithi088 ceph-mon[29151]: Saving service mds.foofs spec with placement count:2 2024-09-16T09:35:04.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:03 smithi088 ceph-mon[29151]: Deploying daemon mds.foofs.smithi078.hiotnn on smithi078 2024-09-16T09:35:04.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:03 smithi088 ceph-mon[29151]: osdmap e63: 8 total, 8 up, 8 in 2024-09-16T09:35:04.665 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:04 smithi078 ceph-mon[23694]: pgmap v218: 65 pgs: 10 creating+peering, 3 active+clean, 52 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:35:04.666 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:04 smithi078 ceph-mon[23694]: osdmap e64: 8 total, 8 up, 8 in 2024-09-16T09:35:05.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:04 smithi088 ceph-mon[29151]: pgmap v218: 65 pgs: 10 creating+peering, 3 active+clean, 52 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:35:05.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:04 smithi088 ceph-mon[29151]: osdmap e64: 8 total, 8 up, 8 in 2024-09-16T09:35:05.397 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-16T09:35:05.397 INFO:teuthology.orchestra.run.smithi078.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-16T09:27:01.706525Z", "last_refresh": "2024-09-16T09:32:08.772847Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-16T09:26:56.078865Z", "last_refresh": "2024-09-16T09:32:08.772928Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-16T09:26:58.846101Z", "last_refresh": "2024-09-16T09:32:08.772992Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-16T09:35:01.647061Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-16T09:35:01.639512Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-16T09:26:54.609387Z", "last_refresh": "2024-09-16T09:32:08.772777Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi078:172.21.15.78=smithi078", "smithi088:172.21.15.88=smithi088"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-16T09:27:44.045718Z", "last_refresh": "2024-09-16T09:32:08.772633Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-16T09:27:00.481998Z", "last_refresh": "2024-09-16T09:32:08.773054Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-16T09:32:08.773195Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-16T09:26:57.619957Z", "last_refresh": "2024-09-16T09:32:08.773134Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-16T09:35:05.914 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:05 smithi078 ceph-mon[23694]: from='client.14534 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T09:35:05.915 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:05 smithi078 ceph-mon[23694]: pgmap v220: 65 pgs: 10 creating+peering, 38 active+clean, 17 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:35:06.040 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-16T09:35:06.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:05 smithi088 ceph-mon[29151]: from='client.14534 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T09:35:06.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:05 smithi088 ceph-mon[29151]: pgmap v220: 65 pgs: 10 creating+peering, 38 active+clean, 17 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-16T09:35:07.041 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-16T09:35:07.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:07 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:07.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:07 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi088.dbunob", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-16T09:35:07.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:07 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi088.dbunob", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-16T09:35:07.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:07 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:35:07.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:07 smithi088 ceph-mon[29151]: Deploying daemon mds.foofs.smithi088.dbunob on smithi088 2024-09-16T09:35:07.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:07 smithi088 ceph-mon[29151]: mds.? [v2:172.21.15.78:6834/245032473,v1:172.21.15.78:6835/245032473] up:boot 2024-09-16T09:35:07.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:07 smithi088 ceph-mon[29151]: daemon mds.foofs.smithi078.hiotnn assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-16T09:35:07.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:07 smithi088 ceph-mon[29151]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-16T09:35:07.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:07 smithi088 ceph-mon[29151]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-16T09:35:07.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:07 smithi088 ceph-mon[29151]: Cluster is now healthy 2024-09-16T09:35:07.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:07 smithi088 ceph-mon[29151]: fsmap foofs:0 1 up:standby 2024-09-16T09:35:07.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:07 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.hiotnn"}]: dispatch 2024-09-16T09:35:07.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:07 smithi088 ceph-mon[29151]: fsmap foofs:1 {0=foofs.smithi078.hiotnn=up:creating} 2024-09-16T09:35:07.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:07 smithi088 ceph-mon[29151]: daemon mds.foofs.smithi078.hiotnn is now active in filesystem foofs as rank 0 2024-09-16T09:35:07.599 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:07 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:07.599 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:07 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi088.dbunob", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-16T09:35:07.600 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:07 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi088.dbunob", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-16T09:35:07.600 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:07 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:35:07.600 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:07 smithi078 ceph-mon[23694]: Deploying daemon mds.foofs.smithi088.dbunob on smithi088 2024-09-16T09:35:07.600 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:07 smithi078 ceph-mon[23694]: mds.? [v2:172.21.15.78:6834/245032473,v1:172.21.15.78:6835/245032473] up:boot 2024-09-16T09:35:07.600 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:07 smithi078 ceph-mon[23694]: daemon mds.foofs.smithi078.hiotnn assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-16T09:35:07.600 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:07 smithi078 ceph-mon[23694]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-16T09:35:07.600 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:07 smithi078 ceph-mon[23694]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-16T09:35:07.600 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:07 smithi078 ceph-mon[23694]: Cluster is now healthy 2024-09-16T09:35:07.600 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:07 smithi078 ceph-mon[23694]: fsmap foofs:0 1 up:standby 2024-09-16T09:35:07.600 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:07 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.hiotnn"}]: dispatch 2024-09-16T09:35:07.600 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:07 smithi078 ceph-mon[23694]: fsmap foofs:1 {0=foofs.smithi078.hiotnn=up:creating} 2024-09-16T09:35:07.600 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:07 smithi078 ceph-mon[23694]: daemon mds.foofs.smithi078.hiotnn is now active in filesystem foofs as rank 0 2024-09-16T09:35:08.513 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:08 smithi088 ceph-mon[29151]: pgmap v221: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 1 op/s 2024-09-16T09:35:08.513 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:08 smithi088 ceph-mon[29151]: mds.? [v2:172.21.15.78:6834/245032473,v1:172.21.15.78:6835/245032473] up:active 2024-09-16T09:35:08.513 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:08 smithi088 ceph-mon[29151]: fsmap foofs:1 {0=foofs.smithi078.hiotnn=up:active} 2024-09-16T09:35:08.676 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:08 smithi078 ceph-mon[23694]: pgmap v221: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 1 op/s 2024-09-16T09:35:08.676 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:08 smithi078 ceph-mon[23694]: mds.? [v2:172.21.15.78:6834/245032473,v1:172.21.15.78:6835/245032473] up:active 2024-09-16T09:35:08.676 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:08 smithi078 ceph-mon[23694]: fsmap foofs:1 {0=foofs.smithi078.hiotnn=up:active} 2024-09-16T09:35:09.111 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-16T09:35:09.111 INFO:teuthology.orchestra.run.smithi078.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-16T09:27:01.706525Z", "last_refresh": "2024-09-16T09:32:08.772847Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-16T09:26:56.078865Z", "last_refresh": "2024-09-16T09:32:08.772928Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-16T09:26:58.846101Z", "last_refresh": "2024-09-16T09:32:08.772992Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-16T09:35:01.647061Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-16T09:35:01.639512Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-16T09:26:54.609387Z", "last_refresh": "2024-09-16T09:32:08.772777Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi078:172.21.15.78=smithi078", "smithi088:172.21.15.88=smithi088"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-16T09:27:44.045718Z", "last_refresh": "2024-09-16T09:32:08.772633Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-16T09:27:00.481998Z", "last_refresh": "2024-09-16T09:32:08.773054Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-16T09:32:08.773195Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-16T09:26:57.619957Z", "last_refresh": "2024-09-16T09:32:08.773134Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-16T09:35:10.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:09 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:10.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:09 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:35:10.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:09 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:35:10.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:09 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:35:10.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:09 smithi088 ceph-mon[29151]: from='client.14546 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T09:35:10.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:09 smithi088 ceph-mon[29151]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 2 op/s 2024-09-16T09:35:10.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:09 smithi088 ceph-mon[29151]: mds.? [v2:172.21.15.88:6832/861752956,v1:172.21.15.88:6833/861752956] up:boot 2024-09-16T09:35:10.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:09 smithi088 ceph-mon[29151]: fsmap foofs:1 {0=foofs.smithi078.hiotnn=up:active} 1 up:standby 2024-09-16T09:35:10.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:09 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi088.dbunob"}]: dispatch 2024-09-16T09:35:10.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:09 smithi088 ceph-mon[29151]: fsmap foofs:1 {0=foofs.smithi078.hiotnn=up:active} 1 up:standby 2024-09-16T09:35:10.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:09 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:10.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:09 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:10.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:09 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:35:10.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:09 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:35:10.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:09 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:35:10.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:09 smithi078 ceph-mon[23694]: from='client.14546 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T09:35:10.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:09 smithi078 ceph-mon[23694]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 2 op/s 2024-09-16T09:35:10.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:09 smithi078 ceph-mon[23694]: mds.? [v2:172.21.15.88:6832/861752956,v1:172.21.15.88:6833/861752956] up:boot 2024-09-16T09:35:10.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:09 smithi078 ceph-mon[23694]: fsmap foofs:1 {0=foofs.smithi078.hiotnn=up:active} 1 up:standby 2024-09-16T09:35:10.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:09 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi088.dbunob"}]: dispatch 2024-09-16T09:35:10.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:09 smithi078 ceph-mon[23694]: fsmap foofs:1 {0=foofs.smithi078.hiotnn=up:active} 1 up:standby 2024-09-16T09:35:10.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:09 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:10.585 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-16T09:35:11.587 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-16T09:35:12.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:11 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:12.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:11 smithi088 ceph-mon[29151]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 3 op/s 2024-09-16T09:35:12.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:11 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:12.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:11 smithi078 ceph-mon[23694]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 3 op/s 2024-09-16T09:35:14.472 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-16T09:35:14.473 INFO:teuthology.orchestra.run.smithi078.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-16T09:27:01.706525Z", "last_refresh": "2024-09-16T09:32:08.772847Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-16T09:26:56.078865Z", "last_refresh": "2024-09-16T09:32:08.772928Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-16T09:26:58.846101Z", "last_refresh": "2024-09-16T09:32:08.772992Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-16T09:35:01.647061Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-16T09:35:01.639512Z", "last_refresh": "2024-09-16T09:35:10.606684Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-16T09:26:54.609387Z", "last_refresh": "2024-09-16T09:32:08.772777Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi078:172.21.15.78=smithi078", "smithi088:172.21.15.88=smithi088"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-16T09:27:44.045718Z", "last_refresh": "2024-09-16T09:32:08.772633Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-16T09:27:00.481998Z", "last_refresh": "2024-09-16T09:32:08.773054Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-16T09:32:08.773195Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-16T09:26:57.619957Z", "last_refresh": "2024-09-16T09:32:08.773134Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-16T09:35:14.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:14 smithi088 ceph-mon[29151]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-16T09:35:14.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:14 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:14.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:14 smithi078 ceph-mon[23694]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-16T09:35:14.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:14 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:15.764 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:15 smithi078 ceph-mon[23694]: from='client.14550 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T09:35:15.787 INFO:tasks.cephadm:mds.foofs has 1/2 2024-09-16T09:35:15.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:15 smithi088 ceph-mon[29151]: from='client.14550 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T09:35:16.788 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-16T09:35:16.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:16 smithi088 ceph-mon[29151]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-16T09:35:16.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:16 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:16.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:16 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:16.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:16 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:35:16.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:16 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:35:16.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:16 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:35:16.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:16 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:16.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:16 smithi078 ceph-mon[23694]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-16T09:35:16.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:16 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:16.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:16 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:16.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:16 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:35:16.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:16 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:35:16.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:16 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:35:16.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:16 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:18.651 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-16T09:35:18.652 INFO:teuthology.orchestra.run.smithi078.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-16T09:27:01.706525Z", "last_refresh": "2024-09-16T09:35:15.924188Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-16T09:26:56.078865Z", "last_refresh": "2024-09-16T09:35:10.606029Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-16T09:26:58.846101Z", "last_refresh": "2024-09-16T09:35:15.924311Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-16T09:35:01.647061Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-16T09:35:01.639512Z", "last_refresh": "2024-09-16T09:35:10.606684Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-16T09:26:54.609387Z", "last_refresh": "2024-09-16T09:35:10.606180Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi078:172.21.15.78=smithi078", "smithi088:172.21.15.88=smithi088"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-16T09:27:44.045718Z", "last_refresh": "2024-09-16T09:35:10.606283Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-16T09:27:00.481998Z", "last_refresh": "2024-09-16T09:35:10.606354Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-16T09:35:10.606421Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-16T09:26:57.619957Z", "last_refresh": "2024-09-16T09:35:15.924430Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-16T09:35:18.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:18 smithi088 ceph-mon[29151]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-16T09:35:18.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:18 smithi078 ceph-mon[23694]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-16T09:35:19.394 INFO:tasks.cephadm:mds.foofs has 2/2 2024-09-16T09:35:19.394 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-16T09:35:19.426 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-16T09:35:19.427 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-09-16T09:35:20.769 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:20 smithi078 ceph-mon[23694]: from='client.14554 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T09:35:20.769 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:20 smithi078 ceph-mon[23694]: pgmap v227: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 511 B/s wr, 3 op/s 2024-09-16T09:35:20.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:20 smithi088 ceph-mon[29151]: from='client.14554 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T09:35:20.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:20 smithi088 ceph-mon[29151]: pgmap v227: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 511 B/s wr, 3 op/s 2024-09-16T09:35:22.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:22 smithi088 ceph-mon[29151]: pgmap v228: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 2 op/s 2024-09-16T09:35:22.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:22 smithi088 ceph-mon[29151]: from='client.14558 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:35:22.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:22 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-16T09:35:22.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:22 smithi078 ceph-mon[23694]: pgmap v228: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 2 op/s 2024-09-16T09:35:22.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:22 smithi078 ceph-mon[23694]: from='client.14558 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:35:22.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:22 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-16T09:35:23.513 INFO:teuthology.orchestra.run.smithi078.stdout:NFS Cluster Created Successfully 2024-09-16T09:35:23.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:23 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-16T09:35:23.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:23 smithi088 ceph-mon[29151]: osdmap e65: 8 total, 8 up, 8 in 2024-09-16T09:35:23.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:23 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-16T09:35:23.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:23 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-16T09:35:23.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:23 smithi078 ceph-mon[23694]: osdmap e65: 8 total, 8 up, 8 in 2024-09-16T09:35:23.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:23 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-16T09:35:24.261 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-09-16T09:35:24.495 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:24 smithi078 ceph-mon[23694]: pgmap v230: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-16T09:35:24.496 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-16T09:35:24.496 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:24 smithi078 ceph-mon[23694]: osdmap e66: 8 total, 8 up, 8 in 2024-09-16T09:35:24.496 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:24 smithi078 ceph-mon[23694]: Saving service nfs.foo spec with placement count:2 2024-09-16T09:35:24.496 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:24.496 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:35:24.496 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:35:24.496 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:35:24.496 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:24.496 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:24.496 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:24 smithi078 ceph-mon[23694]: Creating key for client.nfs.foo.0.0.smithi078.wawosh 2024-09-16T09:35:24.496 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.wawosh", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-16T09:35:24.496 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.wawosh", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-16T09:35:24.497 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:24 smithi078 ceph-mon[23694]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-16T09:35:24.497 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-16T09:35:24.497 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-16T09:35:24.497 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:35:24.497 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-16T09:35:24.497 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-16T09:35:24.497 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.wawosh-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-16T09:35:24.497 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.wawosh-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-16T09:35:24.497 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:24 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:35:24.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:24 smithi088 ceph-mon[29151]: pgmap v230: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-16T09:35:24.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-16T09:35:24.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:24 smithi088 ceph-mon[29151]: osdmap e66: 8 total, 8 up, 8 in 2024-09-16T09:35:24.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:24 smithi088 ceph-mon[29151]: Saving service nfs.foo spec with placement count:2 2024-09-16T09:35:24.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:24.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:35:24.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:35:24.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:35:24.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:24.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:24.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:24 smithi088 ceph-mon[29151]: Creating key for client.nfs.foo.0.0.smithi078.wawosh 2024-09-16T09:35:24.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.wawosh", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-16T09:35:24.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.wawosh", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-16T09:35:24.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:24 smithi088 ceph-mon[29151]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-16T09:35:24.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-16T09:35:24.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-16T09:35:24.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:35:24.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-16T09:35:24.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-16T09:35:24.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.wawosh-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-16T09:35:24.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.wawosh-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-16T09:35:24.803 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:24 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:35:25.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:25 smithi088 ceph-mon[29151]: Rados config object exists: conf-nfs.foo 2024-09-16T09:35:25.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:25 smithi088 ceph-mon[29151]: Creating key for client.nfs.foo.0.0.smithi078.wawosh-rgw 2024-09-16T09:35:25.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:25 smithi088 ceph-mon[29151]: Deploying daemon nfs.foo.0.0.smithi078.wawosh on smithi078 2024-09-16T09:35:25.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:25 smithi088 ceph-mon[29151]: osdmap e67: 8 total, 8 up, 8 in 2024-09-16T09:35:25.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:25 smithi078 ceph-mon[23694]: Rados config object exists: conf-nfs.foo 2024-09-16T09:35:25.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:25 smithi078 ceph-mon[23694]: Creating key for client.nfs.foo.0.0.smithi078.wawosh-rgw 2024-09-16T09:35:25.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:25 smithi078 ceph-mon[23694]: Deploying daemon nfs.foo.0.0.smithi078.wawosh on smithi078 2024-09-16T09:35:25.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:25 smithi078 ceph-mon[23694]: osdmap e67: 8 total, 8 up, 8 in 2024-09-16T09:35:26.750 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:26 smithi078 ceph-mon[23694]: pgmap v233: 97 pgs: 6 creating+activating, 3 creating+peering, 15 unknown, 73 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-16T09:35:26.750 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:26 smithi078 ceph-mon[23694]: mgrmap e20: smithi078.rrfxge(active, since 5m), standbys: smithi088.ncxfdz 2024-09-16T09:35:26.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:26 smithi088 ceph-mon[29151]: pgmap v233: 97 pgs: 6 creating+activating, 3 creating+peering, 15 unknown, 73 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-16T09:35:26.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:26 smithi088 ceph-mon[29151]: mgrmap e20: smithi078.rrfxge(active, since 5m), standbys: smithi088.ncxfdz 2024-09-16T09:35:27.544 INFO:teuthology.orchestra.run.smithi078.stderr:Invalid command: Unexpected argument '--clusterid' 2024-09-16T09:35:27.545 INFO:teuthology.orchestra.run.smithi078.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-09-16T09:35:27.545 INFO:teuthology.orchestra.run.smithi078.stderr:Error EINVAL: invalid command 2024-09-16T09:35:27.887 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:35:27.887 INFO:teuthology.orchestra.run.smithi078.stdout: "bind": "/fake", 2024-09-16T09:35:27.888 INFO:teuthology.orchestra.run.smithi078.stdout: "fs": "foofs", 2024-09-16T09:35:27.888 INFO:teuthology.orchestra.run.smithi078.stdout: "path": "/", 2024-09-16T09:35:27.888 INFO:teuthology.orchestra.run.smithi078.stdout: "cluster": "foo", 2024-09-16T09:35:27.888 INFO:teuthology.orchestra.run.smithi078.stdout: "mode": "RW" 2024-09-16T09:35:27.888 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:35:28.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:28 smithi078 ceph-mon[23694]: pgmap v234: 97 pgs: 6 creating+activating, 3 creating+peering, 88 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-16T09:35:28.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:28 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:28.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:28 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi088.bnccmw", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-16T09:35:28.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:28 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi088.bnccmw", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-16T09:35:28.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:28 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-16T09:35:28.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:28 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-16T09:35:28.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:28 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:35:28.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:28 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-16T09:35:28.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:28 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-16T09:35:28.675 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-09-16T09:35:28.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:28 smithi088 ceph-mon[29151]: pgmap v234: 97 pgs: 6 creating+activating, 3 creating+peering, 88 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-16T09:35:28.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:28 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:28.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:28 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi088.bnccmw", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-16T09:35:28.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:28 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi088.bnccmw", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-16T09:35:28.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:28 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-16T09:35:28.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:28 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-16T09:35:28.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:28 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:35:28.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:28 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-16T09:35:28.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:28 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-16T09:35:29.664 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:29 smithi078 ceph-mon[23694]: Creating key for client.nfs.foo.1.0.smithi088.bnccmw 2024-09-16T09:35:29.664 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:29 smithi078 ceph-mon[23694]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-16T09:35:29.664 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:29 smithi078 ceph-mon[23694]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:35:29.664 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:29 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/trash_purge_schedule"}]: dispatch 2024-09-16T09:35:29.664 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:29 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:35:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:29 smithi088 ceph-mon[29151]: Creating key for client.nfs.foo.1.0.smithi088.bnccmw 2024-09-16T09:35:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:29 smithi088 ceph-mon[29151]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-16T09:35:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:29 smithi088 ceph-mon[29151]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:35:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:29 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/trash_purge_schedule"}]: dispatch 2024-09-16T09:35:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:29 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:35:30.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:30 smithi088 ceph-mon[29151]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1.6 KiB/s wr, 3 op/s 2024-09-16T09:35:30.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:30 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-16T09:35:30.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:30 smithi078 ceph-mon[23694]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1.6 KiB/s wr, 3 op/s 2024-09-16T09:35:30.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:30 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-16T09:35:31.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:31 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-16T09:35:31.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:31 smithi088 ceph-mon[29151]: Rados config object exists: conf-nfs.foo 2024-09-16T09:35:31.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:31 smithi088 ceph-mon[29151]: Creating key for client.nfs.foo.1.0.smithi088.bnccmw-rgw 2024-09-16T09:35:31.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:31 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi088.bnccmw-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-16T09:35:31.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:31 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi088.bnccmw-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-16T09:35:31.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:31 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:35:31.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:31 smithi088 ceph-mon[29151]: Deploying daemon nfs.foo.1.0.smithi088.bnccmw on smithi088 2024-09-16T09:35:31.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:31 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-16T09:35:31.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:31 smithi078 ceph-mon[23694]: Rados config object exists: conf-nfs.foo 2024-09-16T09:35:31.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:31 smithi078 ceph-mon[23694]: Creating key for client.nfs.foo.1.0.smithi088.bnccmw-rgw 2024-09-16T09:35:31.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:31 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi088.bnccmw-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-16T09:35:31.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:31 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi088.bnccmw-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-16T09:35:31.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:31 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:35:31.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:31 smithi078 ceph-mon[23694]: Deploying daemon nfs.foo.1.0.smithi088.bnccmw on smithi088 2024-09-16T09:35:32.720 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:32 smithi088 ceph-mon[29151]: from='client.14594 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:35:32.721 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:32 smithi088 ceph-mon[29151]: pgmap v236: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-09-16T09:35:32.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:32 smithi078 ceph-mon[23694]: from='client.14594 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:35:32.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:32 smithi078 ceph-mon[23694]: pgmap v236: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-09-16T09:35:33.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:33 smithi088 ceph-mon[29151]: from='client.14608 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:35:33.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:33 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:33.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:33 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:35:33.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:33 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:35:33.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:33 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:35:33.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:33 smithi078 ceph-mon[23694]: from='client.14608 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:35:33.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:33 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:33.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:33 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:35:33.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:33 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:35:33.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:33 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:35:34.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:34 smithi078 ceph-mon[23694]: from='client.14624 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:35:34.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:34 smithi078 ceph-mon[23694]: pgmap v237: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-09-16T09:35:34.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:34 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:34.998 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:34 smithi088 ceph-mon[29151]: from='client.14624 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:35:34.999 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:34 smithi088 ceph-mon[29151]: pgmap v237: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-09-16T09:35:34.999 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:34 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:36.767 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:36 smithi078 ceph-mon[23694]: from='client.14628 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:35:36.768 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:36 smithi078 ceph-mon[23694]: pgmap v238: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 2.5 KiB/s wr, 6 op/s 2024-09-16T09:35:36.768 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:36 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:36.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:36 smithi088 ceph-mon[29151]: from='client.14628 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:35:36.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:36 smithi088 ceph-mon[29151]: pgmap v238: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 2.5 KiB/s wr, 6 op/s 2024-09-16T09:35:36.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:36 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:37.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:37 smithi088 ceph-mon[29151]: from='client.14632 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:35:37.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:37 smithi078 ceph-mon[23694]: from='client.14632 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:35:38.568 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo ?:2049 2/2 3s ago 15s count:2 2024-09-16T09:35:38.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:38 smithi088 ceph-mon[29151]: from='client.14636 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:35:38.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:38 smithi088 ceph-mon[29151]: pgmap v239: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-09-16T09:35:38.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:38 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:38.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:38 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:38.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:38 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:38.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:38 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:35:38.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:38 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:35:38.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:38 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:35:38.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:38 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:38.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:38 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:38.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:38 smithi078 ceph-mon[23694]: from='client.14636 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:35:38.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:38 smithi078 ceph-mon[23694]: pgmap v239: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-09-16T09:35:38.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:38 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:38.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:38 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:38.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:38 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:38.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:38 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:35:38.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:38 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:35:38.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:38 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:35:38.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:38 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:38.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:38 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:35:39.241 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-16T09:35:39.252 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-16T09:35:39.252 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-09-16T09:35:39.279 INFO:teuthology.orchestra.run.smithi078.stderr:+ mkdir /mnt/foo 2024-09-16T09:35:39.289 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-09-16T09:35:39.354 INFO:teuthology.orchestra.run.smithi078.stderr:++ hostname 2024-09-16T09:35:39.354 INFO:teuthology.orchestra.run.smithi078.stderr:+ mount -t nfs smithi078:/fake /mnt/foo -o sync 2024-09-16T09:35:39.547 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:39 smithi078 ceph-mon[23694]: from='client.14640 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:35:39.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:39 smithi088 ceph-mon[29151]: from='client.14640 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:35:39.846 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-09-16T09:35:39.911 INFO:teuthology.orchestra.run.smithi078.stderr:+ echo test 2024-09-16T09:35:39.921 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-09-16T09:35:39.985 INFO:teuthology.orchestra.run.smithi078.stderr:+ sync 2024-09-16T09:35:40.250 INFO:teuthology.run_tasks:Running task parallel... 2024-09-16T09:35:40.262 INFO:teuthology.task.parallel:starting parallel... 2024-09-16T09:35:40.263 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-16T09:35:40.263 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-16T09:35:40.264 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-16T09:35:40.264 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -e sha1=66e33d6af1496291b0602ee0a859e68145f17dad -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-16T09:35:40.264 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-16T09:35:40.265 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-09-16T09:35:40.266 INFO:teuthology.task.exec:Executing custom commands... 2024-09-16T09:35:40.266 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-16T09:35:40.266 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-09-16T09:35:40.341 INFO:teuthology.orchestra.run.smithi078.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-16T09:35:40.341 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-16T09:35:40.341 INFO:teuthology.orchestra.run.smithi078.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-09-16T09:35:40.349 INFO:teuthology.orchestra.run.smithi078.stdout:3 of 5 processes prepared for launch 0 sec 2024-09-16T09:35:40.450 INFO:teuthology.orchestra.run.smithi078.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-16T09:35:40.450 INFO:teuthology.orchestra.run.smithi078.stdout:releasing clients 2024-09-16T09:35:40.499 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:40 smithi078 ceph-mon[23694]: pgmap v240: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-09-16T09:35:40.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:40 smithi088 ceph-mon[29151]: pgmap v240: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-09-16T09:35:41.450 INFO:teuthology.orchestra.run.smithi078.stdout: 5 145 31.09 MB/sec warmup 1 sec latency 53.729 ms 2024-09-16T09:35:42.450 INFO:teuthology.orchestra.run.smithi078.stdout: 5 389 34.56 MB/sec warmup 2 sec latency 29.055 ms 2024-09-16T09:35:42.592 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:42 smithi078 ceph-mon[23694]: pgmap v241: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-16T09:35:42.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:42 smithi088 ceph-mon[29151]: pgmap v241: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-16T09:35:43.450 INFO:teuthology.orchestra.run.smithi078.stdout: 5 520 30.09 MB/sec warmup 3 sec latency 30.051 ms 2024-09-16T09:35:43.769 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -e sha1=66e33d6af1496291b0602ee0a859e68145f17dad -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-16T09:35:44.450 INFO:teuthology.orchestra.run.smithi078.stdout: 5 676 26.42 MB/sec warmup 4 sec latency 39.262 ms 2024-09-16T09:35:44.671 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:44 smithi078 ceph-mon[23694]: pgmap v242: 97 pgs: 97 active+clean; 5.2 MiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 398 KiB/s wr, 11 op/s 2024-09-16T09:35:44.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:44 smithi088 ceph-mon[29151]: pgmap v242: 97 pgs: 97 active+clean; 5.2 MiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 398 KiB/s wr, 11 op/s 2024-09-16T09:35:45.450 INFO:teuthology.orchestra.run.smithi078.stdout: 5 1110 22.95 MB/sec warmup 5 sec latency 23.520 ms 2024-09-16T09:35:46.450 INFO:teuthology.orchestra.run.smithi078.stdout: 5 2048 22.15 MB/sec warmup 6 sec latency 22.579 ms 2024-09-16T09:35:46.681 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:46 smithi078 ceph-mon[23694]: pgmap v243: 97 pgs: 97 active+clean; 60 MiB data, 126 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 5.0 MiB/s wr, 268 op/s 2024-09-16T09:35:46.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:46 smithi088 ceph-mon[29151]: pgmap v243: 97 pgs: 97 active+clean; 60 MiB data, 126 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 5.0 MiB/s wr, 268 op/s 2024-09-16T09:35:47.459 INFO:teuthology.orchestra.run.smithi078.stdout: 5 2731 22.12 MB/sec warmup 7 sec latency 32.435 ms 2024-09-16T09:35:48.050 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -e sha1=66e33d6af1496291b0602ee0a859e68145f17dad -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-16T09:35:48.459 INFO:teuthology.orchestra.run.smithi078.stdout: 5 3281 21.82 MB/sec warmup 8 sec latency 47.936 ms 2024-09-16T09:35:48.721 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:48 smithi078 ceph-mon[23694]: pgmap v244: 97 pgs: 97 active+clean; 107 MiB data, 159 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 9.3 MiB/s wr, 366 op/s 2024-09-16T09:35:48.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:48 smithi088 ceph-mon[29151]: pgmap v244: 97 pgs: 97 active+clean; 107 MiB data, 159 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 9.3 MiB/s wr, 366 op/s 2024-09-16T09:35:49.459 INFO:teuthology.orchestra.run.smithi078.stdout: 5 3742 21.80 MB/sec warmup 9 sec latency 35.219 ms 2024-09-16T09:35:50.459 INFO:teuthology.orchestra.run.smithi078.stdout: 5 3909 19.95 MB/sec warmup 10 sec latency 56.629 ms 2024-09-16T09:35:50.500 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:50 smithi078 ceph-mon[23694]: pgmap v245: 97 pgs: 97 active+clean; 114 MiB data, 182 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 10 MiB/s wr, 477 op/s 2024-09-16T09:35:50.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:50 smithi088 ceph-mon[29151]: pgmap v245: 97 pgs: 97 active+clean; 114 MiB data, 182 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 10 MiB/s wr, 477 op/s 2024-09-16T09:35:51.459 INFO:teuthology.orchestra.run.smithi078.stdout: 5 4449 18.52 MB/sec warmup 11 sec latency 44.691 ms 2024-09-16T09:35:52.459 INFO:teuthology.orchestra.run.smithi078.stdout: 5 5002 18.56 MB/sec warmup 12 sec latency 26.450 ms 2024-09-16T09:35:52.521 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:52 smithi078 ceph-mon[23694]: pgmap v246: 97 pgs: 97 active+clean; 122 MiB data, 230 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 12 MiB/s wr, 623 op/s 2024-09-16T09:35:52.581 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -e sha1=66e33d6af1496291b0602ee0a859e68145f17dad -- bash -c 'ceph mgr module enable nfs --force' 2024-09-16T09:35:52.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:52 smithi088 ceph-mon[29151]: pgmap v246: 97 pgs: 97 active+clean; 122 MiB data, 230 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 12 MiB/s wr, 623 op/s 2024-09-16T09:35:53.459 INFO:teuthology.orchestra.run.smithi078.stdout: 5 5732 17.70 MB/sec warmup 13 sec latency 34.495 ms 2024-09-16T09:35:54.459 INFO:teuthology.orchestra.run.smithi078.stdout: 5 6159 17.34 MB/sec warmup 14 sec latency 66.928 ms 2024-09-16T09:35:54.544 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:54 smithi078 ceph-mon[23694]: pgmap v247: 97 pgs: 97 active+clean; 130 MiB data, 244 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 13 MiB/s wr, 723 op/s 2024-09-16T09:35:54.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:54 smithi088 ceph-mon[29151]: pgmap v247: 97 pgs: 97 active+clean; 130 MiB data, 244 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 13 MiB/s wr, 723 op/s 2024-09-16T09:35:55.459 INFO:teuthology.orchestra.run.smithi078.stdout: 5 6634 17.65 MB/sec warmup 15 sec latency 36.539 ms 2024-09-16T09:35:56.459 INFO:teuthology.orchestra.run.smithi078.stdout: 5 7115 17.88 MB/sec warmup 16 sec latency 41.024 ms 2024-09-16T09:35:56.496 INFO:teuthology.orchestra.run.smithi078.stderr:module 'nfs' is already enabled 2024-09-16T09:35:56.580 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:56 smithi078 ceph-mon[23694]: pgmap v248: 97 pgs: 97 active+clean; 134 MiB data, 259 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 14 MiB/s wr, 909 op/s 2024-09-16T09:35:56.581 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:56 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/2790915026' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-16T09:35:56.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:56 smithi088 ceph-mon[29151]: pgmap v248: 97 pgs: 97 active+clean; 134 MiB data, 259 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 14 MiB/s wr, 909 op/s 2024-09-16T09:35:56.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:56 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/2790915026' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-16T09:35:57.459 INFO:teuthology.orchestra.run.smithi078.stdout: 5 7347 17.27 MB/sec warmup 17 sec latency 73.207 ms 2024-09-16T09:35:57.603 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -e sha1=66e33d6af1496291b0602ee0a859e68145f17dad -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-16T09:35:57.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:57 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/2790915026' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-16T09:35:57.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:57 smithi078 ceph-mon[23694]: mgrmap e21: smithi078.rrfxge(active, since 6m), standbys: smithi088.ncxfdz 2024-09-16T09:35:57.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:57 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/2790915026' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-16T09:35:57.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:57 smithi088 ceph-mon[29151]: mgrmap e21: smithi078.rrfxge(active, since 6m), standbys: smithi088.ncxfdz 2024-09-16T09:35:58.459 INFO:teuthology.orchestra.run.smithi078.stdout: 5 7572 16.45 MB/sec warmup 18 sec latency 57.087 ms 2024-09-16T09:35:58.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:35:58 smithi078 ceph-mon[23694]: pgmap v249: 97 pgs: 97 active+clean; 131 MiB data, 292 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 11 MiB/s wr, 736 op/s 2024-09-16T09:35:58.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:35:58 smithi088 ceph-mon[29151]: pgmap v249: 97 pgs: 97 active+clean; 131 MiB data, 292 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 11 MiB/s wr, 736 op/s 2024-09-16T09:35:59.459 INFO:teuthology.orchestra.run.smithi078.stdout: 5 7948 15.73 MB/sec warmup 19 sec latency 52.469 ms 2024-09-16T09:36:00.460 INFO:teuthology.orchestra.run.smithi078.stdout: 5 8238 15.38 MB/sec warmup 20 sec latency 69.640 ms 2024-09-16T09:36:00.627 INFO:teuthology.orchestra.run.smithi078.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:66e33d6af1496291b0602ee0a859e68145f17dad 2024-09-16T09:36:00.722 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:00 smithi078 ceph-mon[23694]: pgmap v250: 97 pgs: 97 active+clean; 138 MiB data, 327 MiB used, 715 GiB / 715 GiB avail; 7.9 KiB/s rd, 7.2 MiB/s wr, 687 op/s 2024-09-16T09:36:00.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:00 smithi088 ceph-mon[29151]: pgmap v250: 97 pgs: 97 active+clean; 138 MiB data, 327 MiB used, 715 GiB / 715 GiB avail; 7.9 KiB/s rd, 7.2 MiB/s wr, 687 op/s 2024-09-16T09:36:01.459 INFO:teuthology.orchestra.run.smithi078.stdout: 5 8619 15.18 MB/sec warmup 21 sec latency 46.823 ms 2024-09-16T09:36:01.758 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:01 smithi078 ceph-mon[23694]: from='client.14660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:66e33d6af1496291b0602ee0a859e68145f17dad", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:36:01.759 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:01 smithi078 ceph-mon[23694]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:66e33d6af1496291b0602ee0a859e68145f17dad 2024-09-16T09:36:01.759 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:01 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:36:01.759 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:01 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:36:01.759 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:01 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:36:01.759 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:01 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:36:01.759 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:01 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:36:01.759 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:01 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:36:01.759 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:01 smithi078 ceph-mon[23694]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:66e33d6af1496291b0602ee0a859e68145f17dad 2024-09-16T09:36:01.759 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:01 smithi078 ceph-mon[23694]: pgmap v251: 97 pgs: 97 active+clean; 147 MiB data, 337 MiB used, 715 GiB / 715 GiB avail; 7.9 KiB/s rd, 7.7 MiB/s wr, 747 op/s 2024-09-16T09:36:02.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:01 smithi088 ceph-mon[29151]: from='client.14660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:66e33d6af1496291b0602ee0a859e68145f17dad", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:36:02.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:01 smithi088 ceph-mon[29151]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:66e33d6af1496291b0602ee0a859e68145f17dad 2024-09-16T09:36:02.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:01 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:36:02.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:01 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:36:02.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:01 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:36:02.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:01 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:36:02.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:01 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:36:02.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:01 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:36:02.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:01 smithi088 ceph-mon[29151]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:66e33d6af1496291b0602ee0a859e68145f17dad 2024-09-16T09:36:02.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:01 smithi088 ceph-mon[29151]: pgmap v251: 97 pgs: 97 active+clean; 147 MiB data, 337 MiB used, 715 GiB / 715 GiB avail; 7.9 KiB/s rd, 7.7 MiB/s wr, 747 op/s 2024-09-16T09:36:02.066 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-16T09:36:02.067 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-16T09:36:02.067 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -e sha1=66e33d6af1496291b0602ee0a859e68145f17dad -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-09-16T09:36:02.460 INFO:teuthology.orchestra.run.smithi078.stdout: 5 9069 14.73 MB/sec warmup 22 sec latency 53.891 ms 2024-09-16T09:36:03.460 INFO:teuthology.orchestra.run.smithi078.stdout: 5 9433 14.33 MB/sec warmup 23 sec latency 52.523 ms 2024-09-16T09:36:04.460 INFO:teuthology.orchestra.run.smithi078.stdout: 5 9889 14.48 MB/sec warmup 24 sec latency 39.554 ms 2024-09-16T09:36:04.597 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:04 smithi078 ceph-mon[23694]: pgmap v252: 97 pgs: 97 active+clean; 144 MiB data, 367 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 6.3 MiB/s wr, 613 op/s 2024-09-16T09:36:04.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:04 smithi088 ceph-mon[29151]: pgmap v252: 97 pgs: 97 active+clean; 144 MiB data, 367 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 6.3 MiB/s wr, 613 op/s 2024-09-16T09:36:05.331 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-16T09:36:05.460 INFO:teuthology.orchestra.run.smithi078.stdout: 5 10465 14.81 MB/sec warmup 25 sec latency 26.588 ms 2024-09-16T09:36:06.353 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T09:36:06.354 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (5m) 28s ago 8m 22.0M - ba2b418f427c 08288e8d0451 2024-09-16T09:36:06.354 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (8m) 28s ago 8m 6405k - 17.2.0 e1d6a67b021e a7184b35ee05 2024-09-16T09:36:06.354 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi088 smithi088 running (6m) 30s ago 6m 6409k - 17.2.0 e1d6a67b021e a9a129cd5d7d 2024-09-16T09:36:06.354 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (5m) 28s ago 6m 54.0M - 8.3.5 dad864ee21e9 53d95a875f02 2024-09-16T09:36:06.354 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.hiotnn smithi078 running (61s) 28s ago 60s 14.9M - 17.2.0 e1d6a67b021e c43db7767ab9 2024-09-16T09:36:06.354 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi088.dbunob smithi088 running (58s) 30s ago 57s 14.5M - 17.2.0 e1d6a67b021e d235f1bfbb05 2024-09-16T09:36:06.354 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rrfxge smithi078 *:9283 running (9m) 28s ago 9m 460M - 17.2.0 e1d6a67b021e 3445d8bfa64d 2024-09-16T09:36:06.354 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi088.ncxfdz smithi088 *:8443,9283 running (6m) 30s ago 6m 415M - 17.2.0 e1d6a67b021e 5971b96856e6 2024-09-16T09:36:06.354 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (9m) 28s ago 9m 49.9M 2048M 17.2.0 e1d6a67b021e c6b35fd1a57a 2024-09-16T09:36:06.354 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi088 smithi088 running (6m) 30s ago 6m 40.0M 2048M 17.2.0 e1d6a67b021e 774a14c00113 2024-09-16T09:36:06.354 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.wawosh smithi078 *:2049 running (39s) 28s ago 38s 51.4M - 4.0 e1d6a67b021e 2544ed9d67f5 2024-09-16T09:36:06.354 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi088.bnccmw smithi088 *:2049 running (33s) 30s ago 33s 48.7M - 4.0 e1d6a67b021e e606adb67049 2024-09-16T09:36:06.355 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (6m) 28s ago 6m 17.8M - 1dbe0e931976 3716a43984ae 2024-09-16T09:36:06.355 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi088 smithi088 *:9100 running (6m) 30s ago 6m 18.9M - 1dbe0e931976 bc997e95eb63 2024-09-16T09:36:06.355 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (5m) 28s ago 5m 45.4M 3216M 17.2.0 e1d6a67b021e 453389fafdff 2024-09-16T09:36:06.355 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (4m) 28s ago 4m 43.8M 3216M 17.2.0 e1d6a67b021e d8b72a22a11b 2024-09-16T09:36:06.355 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (4m) 28s ago 4m 39.9M 3216M 17.2.0 e1d6a67b021e aa521853c9bf 2024-09-16T09:36:06.355 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (4m) 28s ago 4m 40.0M 3216M 17.2.0 e1d6a67b021e db62172fe90c 2024-09-16T09:36:06.355 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi088 running (3m) 30s ago 3m 39.7M 3984M 17.2.0 e1d6a67b021e 6f4e80fe127a 2024-09-16T09:36:06.355 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi088 running (3m) 30s ago 3m 42.1M 3984M 17.2.0 e1d6a67b021e 73e51edb7756 2024-09-16T09:36:06.355 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi088 running (2m) 30s ago 2m 39.9M 3984M 17.2.0 e1d6a67b021e 7c1459b1a2d9 2024-09-16T09:36:06.355 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi088 running (2m) 30s ago 2m 41.3M 3984M 17.2.0 e1d6a67b021e 37f619c2ddf9 2024-09-16T09:36:06.355 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (5m) 28s ago 5m 63.1M - 514e6a882f6e db9476c8a1d8 2024-09-16T09:36:06.460 INFO:teuthology.orchestra.run.smithi078.stdout: 5 10887 14.96 MB/sec warmup 26 sec latency 36.919 ms 2024-09-16T09:36:06.632 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:06 smithi078 ceph-mon[23694]: from='client.14664 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:36:06.632 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:06 smithi078 ceph-mon[23694]: pgmap v253: 97 pgs: 97 active+clean; 138 MiB data, 413 MiB used, 715 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.9 MiB/s wr, 561 op/s 2024-09-16T09:36:06.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:06 smithi088 ceph-mon[29151]: from='client.14664 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:36:06.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:06 smithi088 ceph-mon[29151]: pgmap v253: 97 pgs: 97 active+clean; 138 MiB data, 413 MiB used, 715 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.9 MiB/s wr, 561 op/s 2024-09-16T09:36:06.990 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:36:06.990 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-16T09:36:06.990 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:36:06.990 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:36:06.990 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-16T09:36:06.990 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:36:06.990 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:36:06.990 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-16T09:36:06.990 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-16T09:36:06.990 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:36:06.991 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-16T09:36:06.991 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:36:06.991 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:36:06.991 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-16T09:36:06.991 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-16T09:36:06.991 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-16T09:36:06.991 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:36:07.460 INFO:teuthology.orchestra.run.smithi078.stdout: 5 11416 14.57 MB/sec warmup 27 sec latency 31.933 ms 2024-09-16T09:36:07.521 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:36:07.521 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:66e33d6af1496291b0602ee0a859e68145f17dad", 2024-09-16T09:36:07.521 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-16T09:36:07.521 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-09-16T09:36:07.521 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "", 2024-09-16T09:36:07.521 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:66e33d6af1496291b0602ee0a859e68145f17dad image" 2024-09-16T09:36:07.521 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:36:07.636 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:07 smithi078 ceph-mon[23694]: from='client.24373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:36:07.636 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:07 smithi078 ceph-mon[23694]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:36:07.636 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:07 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/3097558761' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T09:36:07.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:07 smithi088 ceph-mon[29151]: from='client.24373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:36:07.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:07 smithi088 ceph-mon[29151]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:36:07.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:07 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/3097558761' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T09:36:08.194 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-16T09:36:08.460 INFO:teuthology.orchestra.run.smithi078.stdout: 5 11944 14.57 MB/sec warmup 28 sec latency 26.633 ms 2024-09-16T09:36:08.642 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:08 smithi078 ceph-mon[23694]: pgmap v254: 97 pgs: 97 active+clean; 149 MiB data, 450 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 6.4 MiB/s wr, 598 op/s 2024-09-16T09:36:08.643 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:08 smithi078 ceph-mon[23694]: from='client.24383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:36:08.643 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:08 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/1733575145' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T09:36:08.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:08 smithi088 ceph-mon[29151]: pgmap v254: 97 pgs: 97 active+clean; 149 MiB data, 450 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 6.4 MiB/s wr, 598 op/s 2024-09-16T09:36:08.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:08 smithi088 ceph-mon[29151]: from='client.24383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:36:08.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:08 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/1733575145' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T09:36:09.460 INFO:teuthology.orchestra.run.smithi078.stdout: 5 12408 14.42 MB/sec warmup 29 sec latency 47.239 ms 2024-09-16T09:36:10.460 INFO:teuthology.orchestra.run.smithi078.stdout: 5 12868 14.08 MB/sec warmup 30 sec latency 40.321 ms 2024-09-16T09:36:10.670 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:10 smithi078 ceph-mon[23694]: pgmap v255: 97 pgs: 97 active+clean; 157 MiB data, 457 MiB used, 715 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.4 MiB/s wr, 621 op/s 2024-09-16T09:36:10.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:10 smithi088 ceph-mon[29151]: pgmap v255: 97 pgs: 97 active+clean; 157 MiB data, 457 MiB used, 715 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.4 MiB/s wr, 621 op/s 2024-09-16T09:36:11.460 INFO:teuthology.orchestra.run.smithi078.stdout: 5 13071 13.79 MB/sec warmup 31 sec latency 66.231 ms 2024-09-16T09:36:12.460 INFO:teuthology.orchestra.run.smithi078.stdout: 5 13423 13.84 MB/sec warmup 32 sec latency 62.707 ms 2024-09-16T09:36:12.737 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:12 smithi078 ceph-mon[23694]: pgmap v256: 97 pgs: 97 active+clean; 158 MiB data, 470 MiB used, 715 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.6 MiB/s wr, 684 op/s 2024-09-16T09:36:12.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:12 smithi088 ceph-mon[29151]: pgmap v256: 97 pgs: 97 active+clean; 158 MiB data, 470 MiB used, 715 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.6 MiB/s wr, 684 op/s 2024-09-16T09:36:13.467 INFO:teuthology.orchestra.run.smithi078.stdout: 5 14055 14.20 MB/sec warmup 33 sec latency 43.162 ms 2024-09-16T09:36:14.467 INFO:teuthology.orchestra.run.smithi078.stdout: 5 14616 14.31 MB/sec warmup 34 sec latency 18.973 ms 2024-09-16T09:36:14.526 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:14 smithi078 ceph-mon[23694]: pgmap v257: 97 pgs: 97 active+clean; 155 MiB data, 485 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 5.3 MiB/s wr, 544 op/s 2024-09-16T09:36:14.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:14 smithi088 ceph-mon[29151]: pgmap v257: 97 pgs: 97 active+clean; 155 MiB data, 485 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 5.3 MiB/s wr, 544 op/s 2024-09-16T09:36:15.467 INFO:teuthology.orchestra.run.smithi078.stdout: 5 15183 14.07 MB/sec warmup 35 sec latency 38.844 ms 2024-09-16T09:36:16.467 INFO:teuthology.orchestra.run.smithi078.stdout: 5 15395 13.89 MB/sec warmup 36 sec latency 73.691 ms 2024-09-16T09:36:16.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:16 smithi078 ceph-mon[23694]: pgmap v258: 97 pgs: 97 active+clean; 155 MiB data, 512 MiB used, 715 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.0 MiB/s wr, 577 op/s 2024-09-16T09:36:16.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:16 smithi088 ceph-mon[29151]: pgmap v258: 97 pgs: 97 active+clean; 155 MiB data, 512 MiB used, 715 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.0 MiB/s wr, 577 op/s 2024-09-16T09:36:17.467 INFO:teuthology.orchestra.run.smithi078.stdout: 5 15735 13.86 MB/sec warmup 37 sec latency 49.766 ms 2024-09-16T09:36:18.467 INFO:teuthology.orchestra.run.smithi078.stdout: 5 16088 13.53 MB/sec warmup 38 sec latency 62.775 ms 2024-09-16T09:36:18.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:18 smithi078 ceph-mon[23694]: pgmap v259: 97 pgs: 97 active+clean; 166 MiB data, 553 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 688 op/s 2024-09-16T09:36:18.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:18 smithi088 ceph-mon[29151]: pgmap v259: 97 pgs: 97 active+clean; 166 MiB data, 553 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 688 op/s 2024-09-16T09:36:19.467 INFO:teuthology.orchestra.run.smithi078.stdout: 5 16608 13.41 MB/sec warmup 39 sec latency 54.328 ms 2024-09-16T09:36:20.467 INFO:teuthology.orchestra.run.smithi078.stdout: 5 17452 13.95 MB/sec warmup 40 sec latency 57.145 ms 2024-09-16T09:36:20.639 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:20 smithi078 ceph-mon[23694]: pgmap v260: 97 pgs: 97 active+clean; 165 MiB data, 564 MiB used, 715 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.2 MiB/s wr, 559 op/s 2024-09-16T09:36:20.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:20 smithi088 ceph-mon[29151]: pgmap v260: 97 pgs: 97 active+clean; 165 MiB data, 564 MiB used, 715 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.2 MiB/s wr, 559 op/s 2024-09-16T09:36:21.468 INFO:teuthology.orchestra.run.smithi078.stdout: 5 18085 14.19 MB/sec warmup 41 sec latency 21.178 ms 2024-09-16T09:36:22.467 INFO:teuthology.orchestra.run.smithi078.stdout: 5 18789 14.04 MB/sec warmup 42 sec latency 18.091 ms 2024-09-16T09:36:22.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:22 smithi078 ceph-mon[23694]: pgmap v261: 97 pgs: 97 active+clean; 171 MiB data, 595 MiB used, 715 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.4 MiB/s wr, 614 op/s 2024-09-16T09:36:22.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:22 smithi088 ceph-mon[29151]: pgmap v261: 97 pgs: 97 active+clean; 171 MiB data, 595 MiB used, 715 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.4 MiB/s wr, 614 op/s 2024-09-16T09:36:23.467 INFO:teuthology.orchestra.run.smithi078.stdout: 5 19791 14.22 MB/sec warmup 43 sec latency 17.294 ms 2024-09-16T09:36:24.467 INFO:teuthology.orchestra.run.smithi078.stdout: 5 20639 14.50 MB/sec warmup 44 sec latency 19.866 ms 2024-09-16T09:36:24.662 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:24 smithi078 ceph-mon[23694]: pgmap v262: 97 pgs: 97 active+clean; 171 MiB data, 607 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 5.9 MiB/s wr, 523 op/s 2024-09-16T09:36:24.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:24 smithi088 ceph-mon[29151]: pgmap v262: 97 pgs: 97 active+clean; 171 MiB data, 607 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 5.9 MiB/s wr, 523 op/s 2024-09-16T09:36:25.467 INFO:teuthology.orchestra.run.smithi078.stdout: 5 21453 14.94 MB/sec warmup 45 sec latency 18.012 ms 2024-09-16T09:36:26.467 INFO:teuthology.orchestra.run.smithi078.stdout: 5 22035 14.78 MB/sec warmup 46 sec latency 19.074 ms 2024-09-16T09:36:26.716 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:26 smithi078 ceph-mon[23694]: pgmap v263: 97 pgs: 97 active+clean; 184 MiB data, 658 MiB used, 715 GiB / 715 GiB avail; 7.1 KiB/s rd, 8.0 MiB/s wr, 774 op/s 2024-09-16T09:36:26.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:26 smithi088 ceph-mon[29151]: pgmap v263: 97 pgs: 97 active+clean; 184 MiB data, 658 MiB used, 715 GiB / 715 GiB avail; 7.1 KiB/s rd, 8.0 MiB/s wr, 774 op/s 2024-09-16T09:36:27.467 INFO:teuthology.orchestra.run.smithi078.stdout: 5 22561 14.72 MB/sec warmup 47 sec latency 37.760 ms 2024-09-16T09:36:28.467 INFO:teuthology.orchestra.run.smithi078.stdout: 5 23115 14.69 MB/sec warmup 48 sec latency 45.537 ms 2024-09-16T09:36:28.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:28 smithi078 ceph-mon[23694]: pgmap v264: 97 pgs: 97 active+clean; 188 MiB data, 711 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 9.4 MiB/s wr, 907 op/s 2024-09-16T09:36:28.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:28 smithi088 ceph-mon[29151]: pgmap v264: 97 pgs: 97 active+clean; 188 MiB data, 711 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 9.4 MiB/s wr, 907 op/s 2024-09-16T09:36:29.467 INFO:teuthology.orchestra.run.smithi078.stdout: 5 23839 14.68 MB/sec warmup 49 sec latency 21.651 ms 2024-09-16T09:36:29.733 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:29 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [7, 2]}]: dispatch 2024-09-16T09:36:29.733 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:29 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-09-16T09:36:29.733 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:29 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-09-16T09:36:29.734 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:29 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-09-16T09:36:29.734 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:29 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/trash_purge_schedule"}]: dispatch 2024-09-16T09:36:29.734 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:29 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:36:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:29 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [7, 2]}]: dispatch 2024-09-16T09:36:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:29 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-09-16T09:36:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:29 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-09-16T09:36:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:29 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-09-16T09:36:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:29 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/trash_purge_schedule"}]: dispatch 2024-09-16T09:36:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:29 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:36:30.468 INFO:teuthology.orchestra.run.smithi078.stdout: 5 24685 15.05 MB/sec warmup 50 sec latency 29.993 ms 2024-09-16T09:36:30.738 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:30 smithi078 ceph-mon[23694]: pgmap v265: 97 pgs: 97 active+clean; 186 MiB data, 727 MiB used, 715 GiB / 715 GiB avail; 9.4 KiB/s rd, 8.6 MiB/s wr, 815 op/s 2024-09-16T09:36:30.739 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:30 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [7, 2]}]': finished 2024-09-16T09:36:30.739 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:30 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-09-16T09:36:30.739 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:30 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-09-16T09:36:30.739 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:30 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-09-16T09:36:30.739 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:30 smithi078 ceph-mon[23694]: osdmap e68: 8 total, 8 up, 8 in 2024-09-16T09:36:30.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:30 smithi088 ceph-mon[29151]: pgmap v265: 97 pgs: 97 active+clean; 186 MiB data, 727 MiB used, 715 GiB / 715 GiB avail; 9.4 KiB/s rd, 8.6 MiB/s wr, 815 op/s 2024-09-16T09:36:30.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:30 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [7, 2]}]': finished 2024-09-16T09:36:30.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:30 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-09-16T09:36:30.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:30 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-09-16T09:36:30.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:30 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-09-16T09:36:30.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:30 smithi088 ceph-mon[29151]: osdmap e68: 8 total, 8 up, 8 in 2024-09-16T09:36:31.467 INFO:teuthology.orchestra.run.smithi078.stdout: 5 25283 15.17 MB/sec warmup 51 sec latency 18.401 ms 2024-09-16T09:36:31.742 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:31 smithi078 ceph-mon[23694]: osdmap e69: 8 total, 8 up, 8 in 2024-09-16T09:36:32.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:31 smithi088 ceph-mon[29151]: osdmap e69: 8 total, 8 up, 8 in 2024-09-16T09:36:32.467 INFO:teuthology.orchestra.run.smithi078.stdout: 5 25831 14.98 MB/sec warmup 52 sec latency 25.178 ms 2024-09-16T09:36:32.752 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:32 smithi078 ceph-mon[23694]: pgmap v268: 97 pgs: 2 peering, 95 active+clean; 197 MiB data, 744 MiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 12 MiB/s wr, 1.20k op/s 2024-09-16T09:36:32.752 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:32 smithi078 ceph-mon[23694]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-16T09:36:33.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:32 smithi088 ceph-mon[29151]: pgmap v268: 97 pgs: 2 peering, 95 active+clean; 197 MiB data, 744 MiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 12 MiB/s wr, 1.20k op/s 2024-09-16T09:36:33.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:32 smithi088 ceph-mon[29151]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-16T09:36:33.467 INFO:teuthology.orchestra.run.smithi078.stdout: 5 26540 15.10 MB/sec warmup 53 sec latency 34.296 ms 2024-09-16T09:36:34.467 INFO:teuthology.orchestra.run.smithi078.stdout: 5 27381 15.08 MB/sec warmup 54 sec latency 21.179 ms 2024-09-16T09:36:34.764 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:34 smithi078 ceph-mon[23694]: pgmap v269: 97 pgs: 2 peering, 95 active+clean; 196 MiB data, 771 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.5 MiB/s wr, 821 op/s 2024-09-16T09:36:34.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:34 smithi088 ceph-mon[29151]: pgmap v269: 97 pgs: 2 peering, 95 active+clean; 196 MiB data, 771 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.5 MiB/s wr, 821 op/s 2024-09-16T09:36:35.467 INFO:teuthology.orchestra.run.smithi078.stdout: 5 28307 15.50 MB/sec warmup 55 sec latency 21.334 ms 2024-09-16T09:36:36.467 INFO:teuthology.orchestra.run.smithi078.stdout: 5 28956 15.54 MB/sec warmup 56 sec latency 43.219 ms 2024-09-16T09:36:36.776 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:36 smithi078 ceph-mon[23694]: pgmap v270: 97 pgs: 1 peering, 96 active+clean; 204 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 8.7 MiB/s wr, 791 op/s 2024-09-16T09:36:37.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:36 smithi088 ceph-mon[29151]: pgmap v270: 97 pgs: 1 peering, 96 active+clean; 204 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 8.7 MiB/s wr, 791 op/s 2024-09-16T09:36:37.467 INFO:teuthology.orchestra.run.smithi078.stdout: 5 29294 15.33 MB/sec warmup 57 sec latency 51.321 ms 2024-09-16T09:36:37.790 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:37 smithi078 ceph-mon[23694]: pgmap v271: 97 pgs: 97 active+clean; 206 MiB data, 863 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 932 op/s; 0 B/s, 0 objects/s recovering 2024-09-16T09:36:37.790 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:37 smithi078 ceph-mon[23694]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-16T09:36:37.791 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:37 smithi078 ceph-mon[23694]: Cluster is now healthy 2024-09-16T09:36:38.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:37 smithi088 ceph-mon[29151]: pgmap v271: 97 pgs: 97 active+clean; 206 MiB data, 863 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 10 MiB/s wr, 932 op/s; 0 B/s, 0 objects/s recovering 2024-09-16T09:36:38.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:37 smithi088 ceph-mon[29151]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-16T09:36:38.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:37 smithi088 ceph-mon[29151]: Cluster is now healthy 2024-09-16T09:36:38.467 INFO:teuthology.orchestra.run.smithi078.stdout: 5 29534 15.17 MB/sec warmup 58 sec latency 102.778 ms 2024-09-16T09:36:38.589 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-16T09:36:39.353 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T09:36:39.353 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (6m) 61s ago 9m 22.0M - ba2b418f427c 08288e8d0451 2024-09-16T09:36:39.354 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (8m) 61s ago 8m 6405k - 17.2.0 e1d6a67b021e a7184b35ee05 2024-09-16T09:36:39.354 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi088 smithi088 running (6m) 63s ago 6m 6409k - 17.2.0 e1d6a67b021e a9a129cd5d7d 2024-09-16T09:36:39.354 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (5m) 61s ago 7m 54.0M - 8.3.5 dad864ee21e9 53d95a875f02 2024-09-16T09:36:39.354 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.hiotnn smithi078 running (94s) 61s ago 93s 14.9M - 17.2.0 e1d6a67b021e c43db7767ab9 2024-09-16T09:36:39.354 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi088.dbunob smithi088 running (91s) 63s ago 90s 14.5M - 17.2.0 e1d6a67b021e d235f1bfbb05 2024-09-16T09:36:39.354 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rrfxge smithi078 *:9283 running (10m) 61s ago 10m 460M - 17.2.0 e1d6a67b021e 3445d8bfa64d 2024-09-16T09:36:39.354 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi088.ncxfdz smithi088 *:8443,9283 running (6m) 63s ago 6m 415M - 17.2.0 e1d6a67b021e 5971b96856e6 2024-09-16T09:36:39.354 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (10m) 61s ago 10m 49.9M 2048M 17.2.0 e1d6a67b021e c6b35fd1a57a 2024-09-16T09:36:39.354 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi088 smithi088 running (6m) 63s ago 6m 40.0M 2048M 17.2.0 e1d6a67b021e 774a14c00113 2024-09-16T09:36:39.354 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.wawosh smithi078 *:2049 running (72s) 61s ago 71s 51.4M - 4.0 e1d6a67b021e 2544ed9d67f5 2024-09-16T09:36:39.354 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi088.bnccmw smithi088 *:2049 running (66s) 63s ago 66s 48.7M - 4.0 e1d6a67b021e e606adb67049 2024-09-16T09:36:39.354 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (7m) 61s ago 7m 17.8M - 1dbe0e931976 3716a43984ae 2024-09-16T09:36:39.354 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi088 smithi088 *:9100 running (6m) 63s ago 6m 18.9M - 1dbe0e931976 bc997e95eb63 2024-09-16T09:36:39.355 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (6m) 61s ago 6m 45.4M 3216M 17.2.0 e1d6a67b021e 453389fafdff 2024-09-16T09:36:39.355 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (5m) 61s ago 5m 43.8M 3216M 17.2.0 e1d6a67b021e d8b72a22a11b 2024-09-16T09:36:39.355 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (5m) 61s ago 5m 39.9M 3216M 17.2.0 e1d6a67b021e aa521853c9bf 2024-09-16T09:36:39.355 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (4m) 61s ago 4m 40.0M 3216M 17.2.0 e1d6a67b021e db62172fe90c 2024-09-16T09:36:39.355 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi088 running (4m) 63s ago 4m 39.7M 3984M 17.2.0 e1d6a67b021e 6f4e80fe127a 2024-09-16T09:36:39.355 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi088 running (3m) 63s ago 3m 42.1M 3984M 17.2.0 e1d6a67b021e 73e51edb7756 2024-09-16T09:36:39.355 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi088 running (3m) 63s ago 3m 39.9M 3984M 17.2.0 e1d6a67b021e 7c1459b1a2d9 2024-09-16T09:36:39.355 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi088 running (3m) 63s ago 3m 41.3M 3984M 17.2.0 e1d6a67b021e 37f619c2ddf9 2024-09-16T09:36:39.355 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (6m) 61s ago 6m 63.1M - 514e6a882f6e db9476c8a1d8 2024-09-16T09:36:39.467 INFO:teuthology.orchestra.run.smithi078.stdout: 5 29881 15.08 MB/sec warmup 59 sec latency 111.770 ms 2024-09-16T09:36:39.576 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:39 smithi078 ceph-mon[23694]: from='client.24391 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:36:39.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:39 smithi088 ceph-mon[29151]: from='client.24391 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:36:39.826 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:36:39.827 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-16T09:36:39.827 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:36:39.827 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:36:39.827 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-16T09:36:39.827 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:36:39.827 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:36:39.827 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-16T09:36:39.827 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-16T09:36:39.827 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:36:39.827 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-16T09:36:39.827 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:36:39.827 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:36:39.827 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-16T09:36:39.828 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-16T09:36:39.828 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-16T09:36:39.828 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:36:40.317 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:36:40.317 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:66e33d6af1496291b0602ee0a859e68145f17dad", 2024-09-16T09:36:40.317 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-16T09:36:40.317 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-09-16T09:36:40.317 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "", 2024-09-16T09:36:40.317 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:66e33d6af1496291b0602ee0a859e68145f17dad image" 2024-09-16T09:36:40.317 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:36:40.467 INFO:teuthology.orchestra.run.smithi078.stdout: 5 30464 14.98 MB/sec warmup 60 sec latency 49.724 ms 2024-09-16T09:36:40.589 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:40 smithi078 ceph-mon[23694]: from='client.14690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:36:40.590 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:40 smithi078 ceph-mon[23694]: from='client.14694 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:36:40.590 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:40 smithi078 ceph-mon[23694]: pgmap v272: 97 pgs: 97 active+clean; 211 MiB data, 895 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 963 op/s; 0 B/s, 0 objects/s recovering 2024-09-16T09:36:40.590 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:40 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/3258190942' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T09:36:40.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:40 smithi088 ceph-mon[29151]: from='client.14690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:36:40.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:40 smithi088 ceph-mon[29151]: from='client.14694 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:36:40.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:40 smithi088 ceph-mon[29151]: pgmap v272: 97 pgs: 97 active+clean; 211 MiB data, 895 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 963 op/s; 0 B/s, 0 objects/s recovering 2024-09-16T09:36:40.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:40 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/3258190942' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T09:36:40.883 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-16T09:36:41.467 INFO:teuthology.orchestra.run.smithi078.stdout: 5 30925 14.93 MB/sec warmup 61 sec latency 62.440 ms 2024-09-16T09:36:41.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:41 smithi078 ceph-mon[23694]: from='client.14702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:36:41.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:41 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/3272614042' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T09:36:41.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:41 smithi088 ceph-mon[29151]: from='client.14702 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:36:41.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:41 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/3272614042' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T09:36:42.467 INFO:teuthology.orchestra.run.smithi078.stdout: 5 31273 14.91 MB/sec warmup 62 sec latency 81.983 ms 2024-09-16T09:36:42.663 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:42 smithi078 ceph-mon[23694]: pgmap v273: 97 pgs: 97 active+clean; 210 MiB data, 912 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 669 op/s; 0 B/s, 0 objects/s recovering 2024-09-16T09:36:42.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:42 smithi088 ceph-mon[29151]: pgmap v273: 97 pgs: 97 active+clean; 210 MiB data, 912 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 669 op/s; 0 B/s, 0 objects/s recovering 2024-09-16T09:36:43.468 INFO:teuthology.orchestra.run.smithi078.stdout: 5 31821 15.03 MB/sec warmup 63 sec latency 48.025 ms 2024-09-16T09:36:44.459 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:44 smithi078 ceph-mon[23694]: pgmap v274: 97 pgs: 97 active+clean; 208 MiB data, 932 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 7.3 MiB/s wr, 627 op/s; 0 B/s, 0 objects/s recovering 2024-09-16T09:36:44.459 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:44 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:36:44.459 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:44 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:36:44.460 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:44 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:36:44.467 INFO:teuthology.orchestra.run.smithi078.stdout: 5 32195 15.03 MB/sec warmup 64 sec latency 52.479 ms 2024-09-16T09:36:44.548 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:44 smithi088 ceph-mon[29151]: pgmap v274: 97 pgs: 97 active+clean; 208 MiB data, 932 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 7.3 MiB/s wr, 627 op/s; 0 B/s, 0 objects/s recovering 2024-09-16T09:36:44.548 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:44 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:36:44.548 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:44 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:36:44.548 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:44 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:36:45.468 INFO:teuthology.orchestra.run.smithi078.stdout: 5 32494 14.86 MB/sec warmup 65 sec latency 42.604 ms 2024-09-16T09:36:45.639 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:45 smithi078 ceph-mon[23694]: Upgrade: Target is version 19.3.0-4882-g66e33d6a (unknown) 2024-09-16T09:36:45.639 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:45 smithi078 ceph-mon[23694]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f'] 2024-09-16T09:36:45.639 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:45 smithi078 ceph-mon[23694]: Upgrade: Need to upgrade myself (mgr.smithi078.rrfxge) 2024-09-16T09:36:45.639 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:45 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:36:45.639 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:45 smithi078 ceph-mon[23694]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f on smithi088 2024-09-16T09:36:45.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:45 smithi088 ceph-mon[29151]: Upgrade: Target is version 19.3.0-4882-g66e33d6a (unknown) 2024-09-16T09:36:45.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:45 smithi088 ceph-mon[29151]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f'] 2024-09-16T09:36:45.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:45 smithi088 ceph-mon[29151]: Upgrade: Need to upgrade myself (mgr.smithi078.rrfxge) 2024-09-16T09:36:45.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:45 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:36:45.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:45 smithi088 ceph-mon[29151]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f on smithi088 2024-09-16T09:36:46.468 INFO:teuthology.orchestra.run.smithi078.stdout: 5 32841 14.68 MB/sec warmup 66 sec latency 64.338 ms 2024-09-16T09:36:46.688 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:46 smithi078 ceph-mon[23694]: pgmap v275: 97 pgs: 97 active+clean; 220 MiB data, 973 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.5 MiB/s wr, 797 op/s; 0 B/s, 0 objects/s recovering 2024-09-16T09:36:46.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:46 smithi088 ceph-mon[29151]: pgmap v275: 97 pgs: 97 active+clean; 220 MiB data, 973 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.5 MiB/s wr, 797 op/s; 0 B/s, 0 objects/s recovering 2024-09-16T09:36:47.468 INFO:teuthology.orchestra.run.smithi078.stdout: 5 33174 14.58 MB/sec warmup 67 sec latency 104.435 ms 2024-09-16T09:36:48.468 INFO:teuthology.orchestra.run.smithi078.stdout: 5 33859 14.61 MB/sec warmup 68 sec latency 42.554 ms 2024-09-16T09:36:48.702 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:48 smithi078 ceph-mon[23694]: pgmap v276: 97 pgs: 97 active+clean; 225 MiB data, 973 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.8 MiB/s wr, 688 op/s; 0 B/s, 0 objects/s recovering 2024-09-16T09:36:48.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:48 smithi088 ceph-mon[29151]: pgmap v276: 97 pgs: 97 active+clean; 225 MiB data, 973 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.8 MiB/s wr, 688 op/s; 0 B/s, 0 objects/s recovering 2024-09-16T09:36:49.468 INFO:teuthology.orchestra.run.smithi078.stdout: 5 34469 14.58 MB/sec warmup 69 sec latency 28.508 ms 2024-09-16T09:36:50.468 INFO:teuthology.orchestra.run.smithi078.stdout: 5 34904 14.62 MB/sec warmup 70 sec latency 38.538 ms 2024-09-16T09:36:50.746 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:50 smithi078 ceph-mon[23694]: pgmap v277: 97 pgs: 97 active+clean; 223 MiB data, 1010 MiB used, 714 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.6 MiB/s wr, 651 op/s 2024-09-16T09:36:50.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:50 smithi088 ceph-mon[29151]: pgmap v277: 97 pgs: 97 active+clean; 223 MiB data, 1010 MiB used, 714 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.6 MiB/s wr, 651 op/s 2024-09-16T09:36:51.468 INFO:teuthology.orchestra.run.smithi078.stdout: 5 35416 14.70 MB/sec warmup 71 sec latency 48.306 ms 2024-09-16T09:36:52.468 INFO:teuthology.orchestra.run.smithi078.stdout: 5 35712 14.67 MB/sec warmup 72 sec latency 64.566 ms 2024-09-16T09:36:52.537 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:52 smithi078 ceph-mon[23694]: pgmap v278: 97 pgs: 97 active+clean; 224 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.0 MiB/s wr, 593 op/s 2024-09-16T09:36:52.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:52 smithi088 ceph-mon[29151]: pgmap v278: 97 pgs: 97 active+clean; 224 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.0 MiB/s wr, 593 op/s 2024-09-16T09:36:53.468 INFO:teuthology.orchestra.run.smithi078.stdout: 5 36000 14.54 MB/sec warmup 73 sec latency 67.204 ms 2024-09-16T09:36:54.468 INFO:teuthology.orchestra.run.smithi078.stdout: 5 36634 14.48 MB/sec warmup 74 sec latency 63.936 ms 2024-09-16T09:36:54.552 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:54 smithi078 ceph-mon[23694]: pgmap v279: 97 pgs: 97 active+clean; 227 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 5.8 MiB/s wr, 538 op/s 2024-09-16T09:36:54.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:54 smithi088 ceph-mon[29151]: pgmap v279: 97 pgs: 97 active+clean; 227 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 5.8 MiB/s wr, 538 op/s 2024-09-16T09:36:55.468 INFO:teuthology.orchestra.run.smithi078.stdout: 5 37266 14.51 MB/sec warmup 75 sec latency 43.614 ms 2024-09-16T09:36:56.468 INFO:teuthology.orchestra.run.smithi078.stdout: 5 37923 14.45 MB/sec warmup 76 sec latency 41.545 ms 2024-09-16T09:36:56.592 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:56 smithi078 ceph-mon[23694]: pgmap v280: 97 pgs: 97 active+clean; 239 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 7.1 MiB/s wr, 704 op/s 2024-09-16T09:36:56.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:56 smithi088 ceph-mon[29151]: pgmap v280: 97 pgs: 97 active+clean; 239 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 7.1 MiB/s wr, 704 op/s 2024-09-16T09:36:57.468 INFO:teuthology.orchestra.run.smithi078.stdout: 5 38464 14.57 MB/sec warmup 77 sec latency 91.613 ms 2024-09-16T09:36:58.468 INFO:teuthology.orchestra.run.smithi078.stdout: 5 39190 14.75 MB/sec warmup 78 sec latency 33.687 ms 2024-09-16T09:36:58.608 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:36:58 smithi078 ceph-mon[23694]: pgmap v281: 97 pgs: 97 active+clean; 239 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.7 MiB/s wr, 628 op/s 2024-09-16T09:36:58.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:36:58 smithi088 ceph-mon[29151]: pgmap v281: 97 pgs: 97 active+clean; 239 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.7 MiB/s wr, 628 op/s 2024-09-16T09:36:59.468 INFO:teuthology.orchestra.run.smithi078.stdout: 5 39571 14.65 MB/sec warmup 79 sec latency 39.746 ms 2024-09-16T09:37:00.468 INFO:teuthology.orchestra.run.smithi078.stdout: 5 40062 14.56 MB/sec warmup 80 sec latency 42.909 ms 2024-09-16T09:37:00.636 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:00 smithi078 ceph-mon[23694]: pgmap v282: 97 pgs: 97 active+clean; 239 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.6 MiB/s wr, 632 op/s 2024-09-16T09:37:00.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:00 smithi088 ceph-mon[29151]: pgmap v282: 97 pgs: 97 active+clean; 239 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.6 MiB/s wr, 632 op/s 2024-09-16T09:37:01.468 INFO:teuthology.orchestra.run.smithi078.stdout: 5 40572 14.53 MB/sec warmup 81 sec latency 54.065 ms 2024-09-16T09:37:02.469 INFO:teuthology.orchestra.run.smithi078.stdout: 5 41031 14.46 MB/sec warmup 82 sec latency 77.930 ms 2024-09-16T09:37:02.676 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:02 smithi078 ceph-mon[23694]: pgmap v283: 97 pgs: 97 active+clean; 247 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.5 MiB/s wr, 680 op/s 2024-09-16T09:37:02.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:02 smithi088 ceph-mon[29151]: pgmap v283: 97 pgs: 97 active+clean; 247 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.5 MiB/s wr, 680 op/s 2024-09-16T09:37:03.468 INFO:teuthology.orchestra.run.smithi078.stdout: 5 41540 14.42 MB/sec warmup 83 sec latency 44.907 ms 2024-09-16T09:37:04.468 INFO:teuthology.orchestra.run.smithi078.stdout: 5 41933 14.42 MB/sec warmup 84 sec latency 46.799 ms 2024-09-16T09:37:04.478 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:04 smithi078 ceph-mon[23694]: pgmap v284: 97 pgs: 97 active+clean; 253 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.4 MiB/s wr, 679 op/s 2024-09-16T09:37:04.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:04 smithi088 ceph-mon[29151]: pgmap v284: 97 pgs: 97 active+clean; 253 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.4 MiB/s wr, 679 op/s 2024-09-16T09:37:05.468 INFO:teuthology.orchestra.run.smithi078.stdout: 5 42329 14.46 MB/sec warmup 85 sec latency 54.015 ms 2024-09-16T09:37:06.468 INFO:teuthology.orchestra.run.smithi078.stdout: 5 42646 14.42 MB/sec warmup 86 sec latency 59.484 ms 2024-09-16T09:37:06.538 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:06 smithi078 ceph-mon[23694]: pgmap v285: 97 pgs: 97 active+clean; 256 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.3 KiB/s rd, 8.1 MiB/s wr, 839 op/s 2024-09-16T09:37:06.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:06 smithi088 ceph-mon[29151]: pgmap v285: 97 pgs: 97 active+clean; 256 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.3 KiB/s rd, 8.1 MiB/s wr, 839 op/s 2024-09-16T09:37:07.468 INFO:teuthology.orchestra.run.smithi078.stdout: 5 42854 14.35 MB/sec warmup 87 sec latency 66.379 ms 2024-09-16T09:37:08.468 INFO:teuthology.orchestra.run.smithi078.stdout: 5 43079 14.23 MB/sec warmup 88 sec latency 63.626 ms 2024-09-16T09:37:08.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:08 smithi078 ceph-mon[23694]: pgmap v286: 97 pgs: 97 active+clean; 256 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 748 op/s 2024-09-16T09:37:08.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:08 smithi088 ceph-mon[29151]: pgmap v286: 97 pgs: 97 active+clean; 256 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 748 op/s 2024-09-16T09:37:09.471 INFO:teuthology.orchestra.run.smithi078.stdout: 5 43325 14.10 MB/sec warmup 89 sec latency 70.161 ms 2024-09-16T09:37:10.471 INFO:teuthology.orchestra.run.smithi078.stdout: 5 43600 14.00 MB/sec warmup 90 sec latency 62.765 ms 2024-09-16T09:37:10.665 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:10 smithi078 ceph-mon[23694]: pgmap v287: 97 pgs: 97 active+clean; 260 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.9 MiB/s wr, 650 op/s 2024-09-16T09:37:10.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:10 smithi088 ceph-mon[29151]: pgmap v287: 97 pgs: 97 active+clean; 260 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.9 MiB/s wr, 650 op/s 2024-09-16T09:37:11.270 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-16T09:37:11.471 INFO:teuthology.orchestra.run.smithi078.stdout: 5 43870 13.91 MB/sec warmup 91 sec latency 84.636 ms 2024-09-16T09:37:12.031 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T09:37:12.031 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (6m) 94s ago 9m 22.0M - ba2b418f427c 08288e8d0451 2024-09-16T09:37:12.031 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (9m) 94s ago 9m 6405k - 17.2.0 e1d6a67b021e a7184b35ee05 2024-09-16T09:37:12.031 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi088 smithi088 running (7m) 96s ago 7m 6409k - 17.2.0 e1d6a67b021e a9a129cd5d7d 2024-09-16T09:37:12.031 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (6m) 94s ago 7m 54.0M - 8.3.5 dad864ee21e9 53d95a875f02 2024-09-16T09:37:12.031 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.hiotnn smithi078 running (2m) 94s ago 2m 14.9M - 17.2.0 e1d6a67b021e c43db7767ab9 2024-09-16T09:37:12.031 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi088.dbunob smithi088 running (2m) 96s ago 2m 14.5M - 17.2.0 e1d6a67b021e d235f1bfbb05 2024-09-16T09:37:12.031 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rrfxge smithi078 *:9283 running (10m) 94s ago 10m 460M - 17.2.0 e1d6a67b021e 3445d8bfa64d 2024-09-16T09:37:12.031 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi088.ncxfdz smithi088 *:8443,9283 running (7m) 96s ago 7m 415M - 17.2.0 e1d6a67b021e 5971b96856e6 2024-09-16T09:37:12.031 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (10m) 94s ago 10m 49.9M 2048M 17.2.0 e1d6a67b021e c6b35fd1a57a 2024-09-16T09:37:12.032 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi088 smithi088 running (7m) 96s ago 7m 40.0M 2048M 17.2.0 e1d6a67b021e 774a14c00113 2024-09-16T09:37:12.032 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.wawosh smithi078 *:2049 running (105s) 94s ago 104s 51.4M - 4.0 e1d6a67b021e 2544ed9d67f5 2024-09-16T09:37:12.032 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi088.bnccmw smithi088 *:2049 running (99s) 96s ago 99s 48.7M - 4.0 e1d6a67b021e e606adb67049 2024-09-16T09:37:12.032 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (7m) 94s ago 7m 17.8M - 1dbe0e931976 3716a43984ae 2024-09-16T09:37:12.032 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi088 smithi088 *:9100 running (7m) 96s ago 7m 18.9M - 1dbe0e931976 bc997e95eb63 2024-09-16T09:37:12.032 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (6m) 94s ago 6m 45.4M 3216M 17.2.0 e1d6a67b021e 453389fafdff 2024-09-16T09:37:12.032 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (6m) 94s ago 6m 43.8M 3216M 17.2.0 e1d6a67b021e d8b72a22a11b 2024-09-16T09:37:12.032 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (5m) 94s ago 5m 39.9M 3216M 17.2.0 e1d6a67b021e aa521853c9bf 2024-09-16T09:37:12.032 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (5m) 94s ago 5m 40.0M 3216M 17.2.0 e1d6a67b021e db62172fe90c 2024-09-16T09:37:12.032 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi088 running (4m) 96s ago 4m 39.7M 3984M 17.2.0 e1d6a67b021e 6f4e80fe127a 2024-09-16T09:37:12.032 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi088 running (4m) 96s ago 4m 42.1M 3984M 17.2.0 e1d6a67b021e 73e51edb7756 2024-09-16T09:37:12.032 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi088 running (3m) 96s ago 3m 39.9M 3984M 17.2.0 e1d6a67b021e 7c1459b1a2d9 2024-09-16T09:37:12.032 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi088 running (3m) 96s ago 3m 41.3M 3984M 17.2.0 e1d6a67b021e 37f619c2ddf9 2024-09-16T09:37:12.033 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (7m) 94s ago 7m 63.1M - 514e6a882f6e db9476c8a1d8 2024-09-16T09:37:12.471 INFO:teuthology.orchestra.run.smithi078.stdout: 5 44339 13.91 MB/sec warmup 92 sec latency 56.651 ms 2024-09-16T09:37:12.523 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:37:12.523 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-16T09:37:12.523 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:37:12.523 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:37:12.523 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-16T09:37:12.523 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:37:12.523 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:37:12.523 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-16T09:37:12.524 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-16T09:37:12.524 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:37:12.524 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-16T09:37:12.524 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:37:12.524 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:37:12.524 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-16T09:37:12.524 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-16T09:37:12.524 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-16T09:37:12.524 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:37:12.703 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:12 smithi078 ceph-mon[23694]: from='client.14710 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:37:12.703 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:12 smithi078 ceph-mon[23694]: pgmap v288: 97 pgs: 97 active+clean; 262 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.6 MiB/s wr, 649 op/s 2024-09-16T09:37:12.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:12 smithi088 ceph-mon[29151]: from='client.14710 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:37:12.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:12 smithi088 ceph-mon[29151]: pgmap v288: 97 pgs: 97 active+clean; 262 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.6 MiB/s wr, 649 op/s 2024-09-16T09:37:12.901 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:37:12.902 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f", 2024-09-16T09:37:12.902 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-16T09:37:12.902 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-09-16T09:37:12.902 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "0/23 daemons upgraded", 2024-09-16T09:37:12.902 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f image on host smithi088" 2024-09-16T09:37:12.902 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:37:13.471 INFO:teuthology.orchestra.run.smithi078.stdout: 5 44984 13.86 MB/sec warmup 93 sec latency 69.906 ms 2024-09-16T09:37:13.472 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-16T09:37:13.734 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:13 smithi078 ceph-mon[23694]: from='client.24411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:37:13.734 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:13 smithi078 ceph-mon[23694]: from='client.14716 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:37:13.734 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:13 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/1961389583' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T09:37:13.734 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:13 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/1192559368' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T09:37:13.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:13 smithi088 ceph-mon[29151]: from='client.24411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:37:13.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:13 smithi088 ceph-mon[29151]: from='client.14716 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:37:13.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:13 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/1961389583' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T09:37:13.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:13 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/1192559368' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T09:37:14.471 INFO:teuthology.orchestra.run.smithi078.stdout: 5 45691 14.03 MB/sec warmup 94 sec latency 34.423 ms 2024-09-16T09:37:14.741 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:14 smithi078 ceph-mon[23694]: from='client.14724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:37:14.742 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:14 smithi078 ceph-mon[23694]: pgmap v289: 97 pgs: 97 active+clean; 266 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 5.6 MiB/s wr, 555 op/s 2024-09-16T09:37:14.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:14 smithi088 ceph-mon[29151]: from='client.14724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:37:14.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:14 smithi088 ceph-mon[29151]: pgmap v289: 97 pgs: 97 active+clean; 266 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 5.6 MiB/s wr, 555 op/s 2024-09-16T09:37:15.471 INFO:teuthology.orchestra.run.smithi078.stdout: 5 46378 14.14 MB/sec warmup 95 sec latency 33.154 ms 2024-09-16T09:37:16.471 INFO:teuthology.orchestra.run.smithi078.stdout: 5 46548 14.03 MB/sec warmup 96 sec latency 73.754 ms 2024-09-16T09:37:16.795 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:16 smithi078 ceph-mon[23694]: pgmap v290: 97 pgs: 97 active+clean; 268 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.0 MiB/s wr, 664 op/s 2024-09-16T09:37:16.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:16 smithi088 ceph-mon[29151]: pgmap v290: 97 pgs: 97 active+clean; 268 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.0 MiB/s wr, 664 op/s 2024-09-16T09:37:17.471 INFO:teuthology.orchestra.run.smithi078.stdout: 5 46753 13.91 MB/sec warmup 97 sec latency 89.255 ms 2024-09-16T09:37:18.472 INFO:teuthology.orchestra.run.smithi078.stdout: 5 47133 13.83 MB/sec warmup 98 sec latency 71.724 ms 2024-09-16T09:37:18.576 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:18 smithi078 ceph-mon[23694]: pgmap v291: 97 pgs: 97 active+clean; 269 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.8 MiB/s wr, 537 op/s 2024-09-16T09:37:18.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:18 smithi088 ceph-mon[29151]: pgmap v291: 97 pgs: 97 active+clean; 269 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.8 MiB/s wr, 537 op/s 2024-09-16T09:37:19.472 INFO:teuthology.orchestra.run.smithi078.stdout: 5 47645 13.80 MB/sec warmup 99 sec latency 65.156 ms 2024-09-16T09:37:20.472 INFO:teuthology.orchestra.run.smithi078.stdout: 5 48289 13.81 MB/sec warmup 100 sec latency 44.727 ms 2024-09-16T09:37:20.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:20 smithi078 ceph-mon[23694]: pgmap v292: 97 pgs: 97 active+clean; 275 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.7 MiB/s wr, 503 op/s 2024-09-16T09:37:20.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:20 smithi088 ceph-mon[29151]: pgmap v292: 97 pgs: 97 active+clean; 275 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.7 MiB/s wr, 503 op/s 2024-09-16T09:37:21.481 INFO:teuthology.orchestra.run.smithi078.stdout: 5 48741 13.77 MB/sec warmup 101 sec latency 78.973 ms 2024-09-16T09:37:22.481 INFO:teuthology.orchestra.run.smithi078.stdout: 5 49163 13.82 MB/sec warmup 102 sec latency 37.551 ms 2024-09-16T09:37:22.675 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:22 smithi078 ceph-mon[23694]: pgmap v293: 97 pgs: 97 active+clean; 278 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.1 MiB/s wr, 593 op/s 2024-09-16T09:37:22.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:22 smithi088 ceph-mon[29151]: pgmap v293: 97 pgs: 97 active+clean; 278 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.1 MiB/s wr, 593 op/s 2024-09-16T09:37:23.481 INFO:teuthology.orchestra.run.smithi078.stdout: 5 49591 13.82 MB/sec warmup 103 sec latency 68.983 ms 2024-09-16T09:37:24.481 INFO:teuthology.orchestra.run.smithi078.stdout: 5 50104 13.87 MB/sec warmup 104 sec latency 43.046 ms 2024-09-16T09:37:24.704 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:24 smithi078 ceph-mon[23694]: pgmap v294: 97 pgs: 97 active+clean; 278 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.8 MiB/s wr, 596 op/s 2024-09-16T09:37:24.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:24 smithi088 ceph-mon[29151]: pgmap v294: 97 pgs: 97 active+clean; 278 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.8 MiB/s wr, 596 op/s 2024-09-16T09:37:25.481 INFO:teuthology.orchestra.run.smithi078.stdout: 5 50405 13.79 MB/sec warmup 105 sec latency 174.170 ms 2024-09-16T09:37:26.481 INFO:teuthology.orchestra.run.smithi078.stdout: 5 50485 13.67 MB/sec warmup 106 sec latency 154.974 ms 2024-09-16T09:37:26.634 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:26 smithi078 ceph-mon[23694]: pgmap v295: 97 pgs: 97 active+clean; 280 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.6 MiB/s wr, 643 op/s 2024-09-16T09:37:26.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:26 smithi088 ceph-mon[29151]: pgmap v295: 97 pgs: 97 active+clean; 280 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.6 MiB/s wr, 643 op/s 2024-09-16T09:37:27.481 INFO:teuthology.orchestra.run.smithi078.stdout: 5 50537 13.54 MB/sec warmup 107 sec latency 627.577 ms 2024-09-16T09:37:28.481 INFO:teuthology.orchestra.run.smithi078.stdout: 5 50643 13.43 MB/sec warmup 108 sec latency 103.710 ms 2024-09-16T09:37:28.714 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:28 smithi078 ceph-mon[23694]: pgmap v296: 97 pgs: 97 active+clean; 290 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 592 op/s 2024-09-16T09:37:28.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:28 smithi088 ceph-mon[29151]: pgmap v296: 97 pgs: 97 active+clean; 290 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 592 op/s 2024-09-16T09:37:29.481 INFO:teuthology.orchestra.run.smithi078.stdout: 5 50770 13.31 MB/sec warmup 109 sec latency 168.971 ms 2024-09-16T09:37:29.550 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:29 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/trash_purge_schedule"}]: dispatch 2024-09-16T09:37:29.550 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:29 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:37:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:29 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/trash_purge_schedule"}]: dispatch 2024-09-16T09:37:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:29 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:37:30.481 INFO:teuthology.orchestra.run.smithi078.stdout: 5 50926 13.22 MB/sec warmup 110 sec latency 120.521 ms 2024-09-16T09:37:30.576 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:30 smithi078 ceph-mon[23694]: pgmap v297: 97 pgs: 97 active+clean; 287 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.9 MiB/s wr, 539 op/s 2024-09-16T09:37:30.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:30 smithi088 ceph-mon[29151]: pgmap v297: 97 pgs: 97 active+clean; 287 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.9 MiB/s wr, 539 op/s 2024-09-16T09:37:31.481 INFO:teuthology.orchestra.run.smithi078.stdout: 5 51106 13.13 MB/sec warmup 111 sec latency 71.304 ms 2024-09-16T09:37:32.488 INFO:teuthology.orchestra.run.smithi078.stdout: 5 51473 13.15 MB/sec warmup 112 sec latency 86.530 ms 2024-09-16T09:37:32.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:32 smithi078 ceph-mon[23694]: pgmap v298: 97 pgs: 97 active+clean; 285 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.2 MiB/s wr, 529 op/s 2024-09-16T09:37:32.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:32 smithi088 ceph-mon[29151]: pgmap v298: 97 pgs: 97 active+clean; 285 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.2 MiB/s wr, 529 op/s 2024-09-16T09:37:33.488 INFO:teuthology.orchestra.run.smithi078.stdout: 5 52033 13.11 MB/sec warmup 113 sec latency 94.361 ms 2024-09-16T09:37:34.489 INFO:teuthology.orchestra.run.smithi078.stdout: 5 52348 13.06 MB/sec warmup 114 sec latency 70.633 ms 2024-09-16T09:37:34.682 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:34 smithi078 ceph-mon[23694]: pgmap v299: 97 pgs: 97 active+clean; 284 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 4.2 MiB/s wr, 422 op/s 2024-09-16T09:37:34.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:34 smithi088 ceph-mon[29151]: pgmap v299: 97 pgs: 97 active+clean; 284 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 4.2 MiB/s wr, 422 op/s 2024-09-16T09:37:35.488 INFO:teuthology.orchestra.run.smithi078.stdout: 5 52665 13.08 MB/sec warmup 115 sec latency 83.638 ms 2024-09-16T09:37:36.489 INFO:teuthology.orchestra.run.smithi078.stdout: 5 53166 13.11 MB/sec warmup 116 sec latency 90.005 ms 2024-09-16T09:37:36.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:36 smithi078 ceph-mon[23694]: pgmap v300: 97 pgs: 97 active+clean; 287 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 499 op/s 2024-09-16T09:37:36.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:36 smithi088 ceph-mon[29151]: pgmap v300: 97 pgs: 97 active+clean; 287 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 499 op/s 2024-09-16T09:37:37.488 INFO:teuthology.orchestra.run.smithi078.stdout: 5 53634 13.15 MB/sec warmup 117 sec latency 151.630 ms 2024-09-16T09:37:38.489 INFO:teuthology.orchestra.run.smithi078.stdout: 5 53974 13.08 MB/sec warmup 118 sec latency 80.518 ms 2024-09-16T09:37:38.579 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:38 smithi078 ceph-mon[23694]: Upgrade: Updating mgr.smithi088.ncxfdz 2024-09-16T09:37:38.580 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:38 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:37:38.580 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:38 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi088.ncxfdz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T09:37:38.580 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:38 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-16T09:37:38.580 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:38 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:37:38.580 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:38 smithi078 ceph-mon[23694]: Deploying daemon mgr.smithi088.ncxfdz on smithi088 2024-09-16T09:37:38.580 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:38 smithi078 ceph-mon[23694]: pgmap v301: 97 pgs: 97 active+clean; 290 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.1 MiB/s wr, 419 op/s 2024-09-16T09:37:38.656 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:38 smithi088 ceph-mon[29151]: Upgrade: Updating mgr.smithi088.ncxfdz 2024-09-16T09:37:38.656 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:38 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:37:38.656 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:38 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi088.ncxfdz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T09:37:38.656 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:38 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-16T09:37:38.656 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:38 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:37:38.656 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:38 smithi088 ceph-mon[29151]: Deploying daemon mgr.smithi088.ncxfdz on smithi088 2024-09-16T09:37:38.656 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:38 smithi088 ceph-mon[29151]: pgmap v301: 97 pgs: 97 active+clean; 290 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.1 MiB/s wr, 419 op/s 2024-09-16T09:37:39.488 INFO:teuthology.orchestra.run.smithi078.stdout: 5 54235 12.99 MB/sec warmup 119 sec latency 123.940 ms 2024-09-16T09:37:40.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:40 smithi078 ceph-mon[23694]: pgmap v302: 97 pgs: 97 active+clean; 301 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.7 MiB/s wr, 423 op/s 2024-09-16T09:37:40.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:40 smithi088 ceph-mon[29151]: pgmap v302: 97 pgs: 97 active+clean; 301 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.7 MiB/s wr, 423 op/s 2024-09-16T09:37:41.489 INFO:teuthology.orchestra.run.smithi078.stdout: 5 54783 8.03 MB/sec execute 1 sec latency 75.244 ms 2024-09-16T09:37:42.489 INFO:teuthology.orchestra.run.smithi078.stdout: 5 55185 11.03 MB/sec execute 2 sec latency 61.051 ms 2024-09-16T09:37:42.666 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:42 smithi078 ceph-mon[23694]: pgmap v303: 97 pgs: 97 active+clean; 302 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.4 MiB/s wr, 444 op/s 2024-09-16T09:37:42.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:42 smithi088 ceph-mon[29151]: pgmap v303: 97 pgs: 97 active+clean; 302 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.4 MiB/s wr, 444 op/s 2024-09-16T09:37:43.489 INFO:teuthology.orchestra.run.smithi078.stdout: 5 55574 8.69 MB/sec execute 3 sec latency 50.617 ms 2024-09-16T09:37:43.896 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-16T09:37:44.489 INFO:teuthology.orchestra.run.smithi078.stdout: 5 55873 8.56 MB/sec execute 4 sec latency 47.146 ms 2024-09-16T09:37:44.620 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T09:37:44.620 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (7m) 2m ago 10m 22.0M - ba2b418f427c 08288e8d0451 2024-09-16T09:37:44.620 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (10m) 2m ago 10m 6405k - 17.2.0 e1d6a67b021e a7184b35ee05 2024-09-16T09:37:44.620 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi088 smithi088 running (8m) 2m ago 8m 6409k - 17.2.0 e1d6a67b021e a9a129cd5d7d 2024-09-16T09:37:44.621 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (7m) 2m ago 8m 54.0M - 8.3.5 dad864ee21e9 53d95a875f02 2024-09-16T09:37:44.621 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.hiotnn smithi078 running (2m) 2m ago 2m 14.9M - 17.2.0 e1d6a67b021e c43db7767ab9 2024-09-16T09:37:44.621 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi088.dbunob smithi088 running (2m) 2m ago 2m 14.5M - 17.2.0 e1d6a67b021e d235f1bfbb05 2024-09-16T09:37:44.621 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rrfxge smithi078 *:9283 running (11m) 2m ago 11m 460M - 17.2.0 e1d6a67b021e 3445d8bfa64d 2024-09-16T09:37:44.621 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi088.ncxfdz smithi088 *:8443,9283 running (8m) 2m ago 8m 415M - 17.2.0 e1d6a67b021e 5971b96856e6 2024-09-16T09:37:44.621 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (11m) 2m ago 11m 49.9M 2048M 17.2.0 e1d6a67b021e c6b35fd1a57a 2024-09-16T09:37:44.621 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi088 smithi088 running (7m) 2m ago 7m 40.0M 2048M 17.2.0 e1d6a67b021e 774a14c00113 2024-09-16T09:37:44.621 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.wawosh smithi078 *:2049 running (2m) 2m ago 2m 51.4M - 4.0 e1d6a67b021e 2544ed9d67f5 2024-09-16T09:37:44.621 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi088.bnccmw smithi088 *:2049 running (2m) 2m ago 2m 48.7M - 4.0 e1d6a67b021e e606adb67049 2024-09-16T09:37:44.621 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (8m) 2m ago 8m 17.8M - 1dbe0e931976 3716a43984ae 2024-09-16T09:37:44.621 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi088 smithi088 *:9100 running (7m) 2m ago 7m 18.9M - 1dbe0e931976 bc997e95eb63 2024-09-16T09:37:44.621 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (7m) 2m ago 7m 45.4M 3216M 17.2.0 e1d6a67b021e 453389fafdff 2024-09-16T09:37:44.621 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (6m) 2m ago 6m 43.8M 3216M 17.2.0 e1d6a67b021e d8b72a22a11b 2024-09-16T09:37:44.622 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (6m) 2m ago 6m 39.9M 3216M 17.2.0 e1d6a67b021e aa521853c9bf 2024-09-16T09:37:44.622 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (5m) 2m ago 5m 40.0M 3216M 17.2.0 e1d6a67b021e db62172fe90c 2024-09-16T09:37:44.622 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi088 running (5m) 2m ago 5m 39.7M 3984M 17.2.0 e1d6a67b021e 6f4e80fe127a 2024-09-16T09:37:44.622 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi088 running (4m) 2m ago 4m 42.1M 3984M 17.2.0 e1d6a67b021e 73e51edb7756 2024-09-16T09:37:44.622 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi088 running (4m) 2m ago 4m 39.9M 3984M 17.2.0 e1d6a67b021e 7c1459b1a2d9 2024-09-16T09:37:44.622 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi088 running (4m) 2m ago 4m 41.3M 3984M 17.2.0 e1d6a67b021e 37f619c2ddf9 2024-09-16T09:37:44.622 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (7m) 2m ago 7m 63.1M - 514e6a882f6e db9476c8a1d8 2024-09-16T09:37:44.712 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:44 smithi078 ceph-mon[23694]: pgmap v304: 97 pgs: 97 active+clean; 301 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.8 KiB/s rd, 4.4 MiB/s wr, 406 op/s 2024-09-16T09:37:44.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:44 smithi088 ceph-mon[29151]: pgmap v304: 97 pgs: 97 active+clean; 301 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.8 KiB/s rd, 4.4 MiB/s wr, 406 op/s 2024-09-16T09:37:45.059 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:37:45.059 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-16T09:37:45.059 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:37:45.059 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:37:45.059 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-16T09:37:45.059 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:37:45.059 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:37:45.059 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-16T09:37:45.059 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-16T09:37:45.059 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:37:45.059 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-16T09:37:45.060 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:37:45.060 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:37:45.060 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-16T09:37:45.060 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-16T09:37:45.060 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-16T09:37:45.060 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:37:45.491 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:45 smithi078 ceph-mon[23694]: from='client.14732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:37:45.491 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:45 smithi078 ceph-mon[23694]: from='client.14736 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:37:45.491 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:45 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/2237336828' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T09:37:45.497 INFO:teuthology.orchestra.run.smithi078.stdout: 5 56099 8.62 MB/sec execute 5 sec latency 74.632 ms 2024-09-16T09:37:45.556 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:37:45.556 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f", 2024-09-16T09:37:45.557 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-16T09:37:45.557 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-09-16T09:37:45.557 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "0/23 daemons upgraded", 2024-09-16T09:37:45.557 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading mgr daemons" 2024-09-16T09:37:45.557 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:37:45.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:45 smithi088 ceph-mon[29151]: from='client.14732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:37:45.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:45 smithi088 ceph-mon[29151]: from='client.14736 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:37:45.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:45 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/2237336828' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T09:37:46.096 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-16T09:37:46.497 INFO:teuthology.orchestra.run.smithi078.stdout: 5 56701 11.10 MB/sec execute 6 sec latency 43.442 ms 2024-09-16T09:37:46.500 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:46 smithi078 ceph-mon[23694]: from='client.14740 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:37:46.500 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:46 smithi078 ceph-mon[23694]: pgmap v305: 97 pgs: 97 active+clean; 300 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.0 MiB/s wr, 491 op/s 2024-09-16T09:37:46.501 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:46 smithi078 ceph-mon[23694]: from='client.14748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:37:46.501 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:46 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/2146693225' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T09:37:46.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:46 smithi088 ceph-mon[29151]: from='client.14740 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:37:46.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:46 smithi088 ceph-mon[29151]: pgmap v305: 97 pgs: 97 active+clean; 300 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.0 MiB/s wr, 491 op/s 2024-09-16T09:37:46.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:46 smithi088 ceph-mon[29151]: from='client.14748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:37:46.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:46 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/2146693225' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T09:37:47.497 INFO:teuthology.orchestra.run.smithi078.stdout: 5 57156 11.78 MB/sec execute 7 sec latency 35.985 ms 2024-09-16T09:37:47.516 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:47 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:37:47.517 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:47 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:37:47.517 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:47 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:37:47.517 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:47 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:37:47.517 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:47 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:37:47.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:47 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:37:47.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:47 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:37:47.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:47 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:37:47.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:47 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:37:47.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:47 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:37:48.498 INFO:teuthology.orchestra.run.smithi078.stdout: 5 57797 11.43 MB/sec execute 8 sec latency 19.561 ms 2024-09-16T09:37:48.524 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:48 smithi078 ceph-mon[23694]: pgmap v306: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.4 MiB/s wr, 502 op/s 2024-09-16T09:37:48.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:48 smithi088 ceph-mon[29151]: pgmap v306: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.4 MiB/s wr, 502 op/s 2024-09-16T09:37:49.497 INFO:teuthology.orchestra.run.smithi078.stdout: 5 58247 11.38 MB/sec execute 9 sec latency 118.326 ms 2024-09-16T09:37:50.497 INFO:teuthology.orchestra.run.smithi078.stdout: 5 58609 11.59 MB/sec execute 10 sec latency 68.611 ms 2024-09-16T09:37:50.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:50 smithi078 ceph-mon[23694]: pgmap v307: 97 pgs: 97 active+clean; 311 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.5 MiB/s wr, 544 op/s 2024-09-16T09:37:50.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:50 smithi088 ceph-mon[29151]: pgmap v307: 97 pgs: 97 active+clean; 311 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.5 MiB/s wr, 544 op/s 2024-09-16T09:37:51.497 INFO:teuthology.orchestra.run.smithi078.stdout: 5 59099 11.00 MB/sec execute 11 sec latency 50.732 ms 2024-09-16T09:37:52.498 INFO:teuthology.orchestra.run.smithi078.stdout: 5 59629 11.46 MB/sec execute 12 sec latency 30.602 ms 2024-09-16T09:37:52.661 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:52 smithi078 ceph-mon[23694]: pgmap v308: 97 pgs: 97 active+clean; 314 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.6 MiB/s wr, 499 op/s 2024-09-16T09:37:52.661 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:52 smithi078 ceph-mon[23694]: Standby manager daemon smithi088.ncxfdz restarted 2024-09-16T09:37:52.661 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:52 smithi078 ceph-mon[23694]: Standby manager daemon smithi088.ncxfdz started 2024-09-16T09:37:52.661 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:52 smithi078 ceph-mon[23694]: from='mgr.? 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi088.ncxfdz/crt"}]: dispatch 2024-09-16T09:37:52.661 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:52 smithi078 ceph-mon[23694]: from='mgr.? 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-16T09:37:52.661 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:52 smithi078 ceph-mon[23694]: from='mgr.? 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi088.ncxfdz/key"}]: dispatch 2024-09-16T09:37:52.661 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:52 smithi078 ceph-mon[23694]: from='mgr.? 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-16T09:37:52.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:52 smithi088 ceph-mon[29151]: pgmap v308: 97 pgs: 97 active+clean; 314 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.6 MiB/s wr, 499 op/s 2024-09-16T09:37:52.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:52 smithi088 ceph-mon[29151]: Standby manager daemon smithi088.ncxfdz restarted 2024-09-16T09:37:52.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:52 smithi088 ceph-mon[29151]: Standby manager daemon smithi088.ncxfdz started 2024-09-16T09:37:52.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:52 smithi088 ceph-mon[29151]: from='mgr.? 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi088.ncxfdz/crt"}]: dispatch 2024-09-16T09:37:52.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:52 smithi088 ceph-mon[29151]: from='mgr.? 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-16T09:37:52.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:52 smithi088 ceph-mon[29151]: from='mgr.? 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi088.ncxfdz/key"}]: dispatch 2024-09-16T09:37:52.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:52 smithi088 ceph-mon[29151]: from='mgr.? 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-16T09:37:53.497 INFO:teuthology.orchestra.run.smithi078.stdout: 5 60186 12.19 MB/sec execute 13 sec latency 32.368 ms 2024-09-16T09:37:53.635 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:53 smithi078 ceph-mon[23694]: mgrmap e22: smithi078.rrfxge(active, since 8m), standbys: smithi088.ncxfdz 2024-09-16T09:37:53.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:53 smithi088 ceph-mon[29151]: mgrmap e22: smithi078.rrfxge(active, since 8m), standbys: smithi088.ncxfdz 2024-09-16T09:37:54.497 INFO:teuthology.orchestra.run.smithi078.stdout: 5 60654 12.61 MB/sec execute 14 sec latency 68.873 ms 2024-09-16T09:37:54.697 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:54 smithi078 ceph-mon[23694]: pgmap v309: 97 pgs: 97 active+clean; 311 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.1 MiB/s wr, 491 op/s 2024-09-16T09:37:54.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:54 smithi088 ceph-mon[29151]: pgmap v309: 97 pgs: 97 active+clean; 311 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.1 MiB/s wr, 491 op/s 2024-09-16T09:37:55.497 INFO:teuthology.orchestra.run.smithi078.stdout: 5 61074 12.34 MB/sec execute 15 sec latency 45.994 ms 2024-09-16T09:37:56.497 INFO:teuthology.orchestra.run.smithi078.stdout: 5 61546 11.91 MB/sec execute 16 sec latency 47.999 ms 2024-09-16T09:37:56.757 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:56 smithi078 ceph-mon[23694]: pgmap v310: 97 pgs: 97 active+clean; 318 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.4 MiB/s wr, 709 op/s 2024-09-16T09:37:56.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:56 smithi088 ceph-mon[29151]: pgmap v310: 97 pgs: 97 active+clean; 318 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.4 MiB/s wr, 709 op/s 2024-09-16T09:37:57.498 INFO:teuthology.orchestra.run.smithi078.stdout: 5 61839 11.69 MB/sec execute 17 sec latency 67.493 ms 2024-09-16T09:37:58.497 INFO:teuthology.orchestra.run.smithi078.stdout: 5 62376 11.88 MB/sec execute 18 sec latency 38.013 ms 2024-09-16T09:37:58.802 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:58 smithi078 ceph-mon[23694]: pgmap v311: 97 pgs: 97 active+clean; 325 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 677 op/s 2024-09-16T09:37:58.802 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:58 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:37:58.803 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:37:58 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:37:58.874 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:58 smithi088 ceph-mon[29151]: pgmap v311: 97 pgs: 97 active+clean; 325 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 677 op/s 2024-09-16T09:37:58.874 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:58 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:37:58.874 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:37:58 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:37:59.498 INFO:teuthology.orchestra.run.smithi078.stdout: 5 62897 11.75 MB/sec execute 19 sec latency 51.747 ms 2024-09-16T09:38:00.502 INFO:teuthology.orchestra.run.smithi078.stdout: 5 63363 12.12 MB/sec execute 20 sec latency 43.830 ms 2024-09-16T09:38:00.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:00 smithi078 ceph-mon[23694]: pgmap v312: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.9 MiB/s wr, 673 op/s 2024-09-16T09:38:00.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:00 smithi088 ceph-mon[29151]: pgmap v312: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.9 MiB/s wr, 673 op/s 2024-09-16T09:38:01.503 INFO:teuthology.orchestra.run.smithi078.stdout: 5 64047 12.71 MB/sec execute 21 sec latency 76.314 ms 2024-09-16T09:38:02.504 INFO:teuthology.orchestra.run.smithi078.stdout: 5 64365 12.58 MB/sec execute 22 sec latency 43.291 ms 2024-09-16T09:38:02.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:02 smithi078 ceph-mon[23694]: pgmap v313: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 7.1 MiB/s wr, 741 op/s 2024-09-16T09:38:02.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:02 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:38:02.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:02 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:38:02.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:02 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:38:02.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:02 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:38:02.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:02 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mgr fail", "who": "smithi078.rrfxge"}]: dispatch 2024-09-16T09:38:02.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:02 smithi078 ceph-mon[23694]: osdmap e70: 8 total, 8 up, 8 in 2024-09-16T09:38:02.840 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:02 smithi088 ceph-mon[29151]: pgmap v313: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 7.1 MiB/s wr, 741 op/s 2024-09-16T09:38:02.840 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:02 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:38:02.840 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:02 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:38:02.840 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:02 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' 2024-09-16T09:38:02.840 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:02 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:38:02.841 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:02 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mgr fail", "who": "smithi078.rrfxge"}]: dispatch 2024-09-16T09:38:02.841 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:02 smithi088 ceph-mon[29151]: osdmap e70: 8 total, 8 up, 8 in 2024-09-16T09:38:03.508 INFO:teuthology.orchestra.run.smithi078.stdout: 5 64691 12.22 MB/sec execute 23 sec latency 55.143 ms 2024-09-16T09:38:03.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:03 smithi078 ceph-mon[23694]: Upgrade: Need to upgrade myself (mgr.smithi078.rrfxge) 2024-09-16T09:38:03.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:03 smithi078 ceph-mon[23694]: Failing over to other MGR 2024-09-16T09:38:03.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:03 smithi078 ceph-mon[23694]: Standby manager daemon smithi078.rrfxge started 2024-09-16T09:38:03.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:03 smithi078 ceph-mon[23694]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "mgr fail", "who": "smithi078.rrfxge"}]': finished 2024-09-16T09:38:03.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:03 smithi078 ceph-mon[23694]: mgrmap e23: smithi088.ncxfdz(active, starting, since 1.00118s), standbys: smithi078.rrfxge 2024-09-16T09:38:03.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:03 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-16T09:38:03.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:03 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "mon metadata", "id": "smithi088"}]: dispatch 2024-09-16T09:38:03.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:03 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.hiotnn"}]: dispatch 2024-09-16T09:38:03.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:03 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi088.dbunob"}]: dispatch 2024-09-16T09:38:03.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:03 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "mgr metadata", "who": "smithi088.ncxfdz", "id": "smithi088.ncxfdz"}]: dispatch 2024-09-16T09:38:03.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:03 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "mgr metadata", "who": "smithi078.rrfxge", "id": "smithi078.rrfxge"}]: dispatch 2024-09-16T09:38:03.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:03 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T09:38:03.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:03 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T09:38:03.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:03 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T09:38:03.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:03 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T09:38:03.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:03 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T09:38:03.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:03 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T09:38:03.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:03 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T09:38:03.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:03 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T09:38:03.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:03 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-16T09:38:03.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:03 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-16T09:38:03.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:03 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-16T09:38:03.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:03 smithi078 ceph-mon[23694]: Manager daemon smithi088.ncxfdz is now available 2024-09-16T09:38:04.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:03 smithi088 ceph-mon[29151]: Upgrade: Need to upgrade myself (mgr.smithi078.rrfxge) 2024-09-16T09:38:04.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:03 smithi088 ceph-mon[29151]: Failing over to other MGR 2024-09-16T09:38:04.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:03 smithi088 ceph-mon[29151]: Standby manager daemon smithi078.rrfxge started 2024-09-16T09:38:04.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:03 smithi088 ceph-mon[29151]: from='mgr.14252 172.21.15.78:0/4255737338' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "mgr fail", "who": "smithi078.rrfxge"}]': finished 2024-09-16T09:38:04.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:03 smithi088 ceph-mon[29151]: mgrmap e23: smithi088.ncxfdz(active, starting, since 1.00118s), standbys: smithi078.rrfxge 2024-09-16T09:38:04.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:03 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-16T09:38:04.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:03 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "mon metadata", "id": "smithi088"}]: dispatch 2024-09-16T09:38:04.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:03 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.hiotnn"}]: dispatch 2024-09-16T09:38:04.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:03 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi088.dbunob"}]: dispatch 2024-09-16T09:38:04.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:03 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "mgr metadata", "who": "smithi088.ncxfdz", "id": "smithi088.ncxfdz"}]: dispatch 2024-09-16T09:38:04.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:03 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "mgr metadata", "who": "smithi078.rrfxge", "id": "smithi078.rrfxge"}]: dispatch 2024-09-16T09:38:04.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:03 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T09:38:04.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:03 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T09:38:04.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:03 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T09:38:04.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:03 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T09:38:04.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:03 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T09:38:04.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:03 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T09:38:04.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:03 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T09:38:04.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:03 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T09:38:04.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:03 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-16T09:38:04.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:03 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-16T09:38:04.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:03 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-16T09:38:04.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:03 smithi088 ceph-mon[29151]: Manager daemon smithi088.ncxfdz is now available 2024-09-16T09:38:04.513 INFO:teuthology.orchestra.run.smithi078.stdout: 5 65023 11.83 MB/sec execute 24 sec latency 70.771 ms 2024-09-16T09:38:04.972 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:04 smithi078 ceph-mon[23694]: mgrmap e24: smithi088.ncxfdz(active, since 2s), standbys: smithi078.rrfxge 2024-09-16T09:38:04.972 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:04 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:04.972 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:04 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:04.972 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:04 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:04.972 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:04 smithi078 ceph-mon[23694]: Checking for cert/key for grafana.smithi078 2024-09-16T09:38:04.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:04 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:04.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:04 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:04.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:04 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:38:04.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:04 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T09:38:04.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:04 smithi078 ceph-mon[23694]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T09:38:04.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:04 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T09:38:04.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:04 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi088.ncxfdz/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:38:04.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:04 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi088.ncxfdz/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:38:04.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:04 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi088.ncxfdz/trash_purge_schedule"}]: dispatch 2024-09-16T09:38:04.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:04 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi088.ncxfdz/trash_purge_schedule"}]: dispatch 2024-09-16T09:38:05.230 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:04 smithi088 ceph-mon[29151]: mgrmap e24: smithi088.ncxfdz(active, since 2s), standbys: smithi078.rrfxge 2024-09-16T09:38:05.230 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:04 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:05.230 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:04 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:05.230 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:04 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:05.230 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:04 smithi088 ceph-mon[29151]: Checking for cert/key for grafana.smithi078 2024-09-16T09:38:05.230 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:04 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:05.230 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:04 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:05.230 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:04 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:38:05.230 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:04 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T09:38:05.231 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:04 smithi088 ceph-mon[29151]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T09:38:05.231 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:04 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T09:38:05.231 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:04 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi088.ncxfdz/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:38:05.231 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:04 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi088.ncxfdz/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:38:05.231 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:04 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi088.ncxfdz/trash_purge_schedule"}]: dispatch 2024-09-16T09:38:05.231 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:04 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi088.ncxfdz/trash_purge_schedule"}]: dispatch 2024-09-16T09:38:05.513 INFO:teuthology.orchestra.run.smithi078.stdout: 5 65203 11.52 MB/sec execute 25 sec latency 101.524 ms 2024-09-16T09:38:06.010 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:05 smithi078 ceph-mon[23694]: Deploying cephadm binary to smithi088 2024-09-16T09:38:06.010 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:05 smithi078 ceph-mon[23694]: Deploying cephadm binary to smithi078 2024-09-16T09:38:06.010 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:05 smithi078 ceph-mon[23694]: pgmap v4: 97 pgs: 97 active+clean; 342 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-16T09:38:06.010 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:05 smithi078 ceph-mon[23694]: [16/Sep/2024:09:38:04] ENGINE Bus STARTING 2024-09-16T09:38:06.010 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:05 smithi078 ceph-mon[23694]: [16/Sep/2024:09:38:05] ENGINE Serving on http://172.21.15.88:8765 2024-09-16T09:38:06.010 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:05 smithi078 ceph-mon[23694]: [16/Sep/2024:09:38:05] ENGINE Serving on https://172.21.15.88:7150 2024-09-16T09:38:06.010 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:05 smithi078 ceph-mon[23694]: [16/Sep/2024:09:38:05] ENGINE Bus STARTED 2024-09-16T09:38:06.010 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:05 smithi078 ceph-mon[23694]: [16/Sep/2024:09:38:05] ENGINE Error in HTTPServer.serve 2024-09-16T09:38:06.010 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Traceback (most recent call last): 2024-09-16T09:38:06.010 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-16T09:38:06.010 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self._connections.run(self.expiration_interval) 2024-09-16T09:38:06.011 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-16T09:38:06.011 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self._run(expiration_interval) 2024-09-16T09:38:06.011 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-16T09:38:06.011 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-16T09:38:06.011 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-16T09:38:06.011 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-16T09:38:06.011 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-16T09:38:06.011 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: s = self.context.wrap_socket( 2024-09-16T09:38:06.011 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-16T09:38:06.011 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: return self.sslsocket_class._create( 2024-09-16T09:38:06.011 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-16T09:38:06.011 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self.do_handshake() 2024-09-16T09:38:06.011 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-16T09:38:06.012 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self._sslobj.do_handshake() 2024-09-16T09:38:06.012 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-16T09:38:06.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:05 smithi088 ceph-mon[29151]: Deploying cephadm binary to smithi088 2024-09-16T09:38:06.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:05 smithi088 ceph-mon[29151]: Deploying cephadm binary to smithi078 2024-09-16T09:38:06.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:05 smithi088 ceph-mon[29151]: pgmap v4: 97 pgs: 97 active+clean; 342 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-16T09:38:06.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:05 smithi088 ceph-mon[29151]: [16/Sep/2024:09:38:04] ENGINE Bus STARTING 2024-09-16T09:38:06.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:05 smithi088 ceph-mon[29151]: [16/Sep/2024:09:38:05] ENGINE Serving on http://172.21.15.88:8765 2024-09-16T09:38:06.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:05 smithi088 ceph-mon[29151]: [16/Sep/2024:09:38:05] ENGINE Serving on https://172.21.15.88:7150 2024-09-16T09:38:06.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:05 smithi088 ceph-mon[29151]: [16/Sep/2024:09:38:05] ENGINE Bus STARTED 2024-09-16T09:38:06.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:05 smithi088 ceph-mon[29151]: [16/Sep/2024:09:38:05] ENGINE Error in HTTPServer.serve 2024-09-16T09:38:06.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Traceback (most recent call last): 2024-09-16T09:38:06.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-16T09:38:06.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: self._connections.run(self.expiration_interval) 2024-09-16T09:38:06.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-16T09:38:06.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: self._run(expiration_interval) 2024-09-16T09:38:06.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-16T09:38:06.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-16T09:38:06.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-16T09:38:06.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-16T09:38:06.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-16T09:38:06.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: s = self.context.wrap_socket( 2024-09-16T09:38:06.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-16T09:38:06.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: return self.sslsocket_class._create( 2024-09-16T09:38:06.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-16T09:38:06.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: self.do_handshake() 2024-09-16T09:38:06.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-16T09:38:06.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: self._sslobj.do_handshake() 2024-09-16T09:38:06.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-16T09:38:06.513 INFO:teuthology.orchestra.run.smithi078.stdout: 5 65473 11.44 MB/sec execute 26 sec latency 61.217 ms 2024-09-16T09:38:07.048 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:06 smithi078 ceph-mon[23694]: mgrmap e25: smithi088.ncxfdz(active, since 4s), standbys: smithi078.rrfxge 2024-09-16T09:38:07.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:06 smithi088 ceph-mon[29151]: mgrmap e25: smithi088.ncxfdz(active, since 4s), standbys: smithi078.rrfxge 2024-09-16T09:38:07.513 INFO:teuthology.orchestra.run.smithi078.stdout: 5 65816 11.43 MB/sec execute 27 sec latency 62.160 ms 2024-09-16T09:38:08.044 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:07 smithi078 ceph-mon[23694]: pgmap v5: 97 pgs: 97 active+clean; 342 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-16T09:38:08.172 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:07 smithi088 ceph-mon[29151]: pgmap v5: 97 pgs: 97 active+clean; 342 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-16T09:38:08.513 INFO:teuthology.orchestra.run.smithi078.stdout: 5 66349 11.25 MB/sec execute 28 sec latency 62.161 ms 2024-09-16T09:38:09.049 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:08 smithi078 ceph-mon[23694]: mgrmap e26: smithi088.ncxfdz(active, since 6s), standbys: smithi078.rrfxge 2024-09-16T09:38:09.227 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:08 smithi088 ceph-mon[29151]: mgrmap e26: smithi088.ncxfdz(active, since 6s), standbys: smithi078.rrfxge 2024-09-16T09:38:09.513 INFO:teuthology.orchestra.run.smithi078.stdout: 5 66681 11.31 MB/sec execute 29 sec latency 54.415 ms 2024-09-16T09:38:10.513 INFO:teuthology.orchestra.run.smithi078.stdout: 5 67142 11.56 MB/sec execute 30 sec latency 44.382 ms 2024-09-16T09:38:10.590 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:10 smithi078 ceph-mon[23694]: pgmap v6: 97 pgs: 97 active+clean; 342 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-16T09:38:10.591 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:10 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:10.591 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:10 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:10.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:10 smithi088 ceph-mon[29151]: pgmap v6: 97 pgs: 97 active+clean; 342 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-16T09:38:10.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:10 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:10.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:10 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:11.513 INFO:teuthology.orchestra.run.smithi078.stdout: 5 67591 11.63 MB/sec execute 31 sec latency 42.436 ms 2024-09-16T09:38:11.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:11 smithi078 ceph-mon[23694]: Standby manager daemon smithi078.rrfxge restarted 2024-09-16T09:38:11.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:11 smithi078 ceph-mon[23694]: Standby manager daemon smithi078.rrfxge started 2024-09-16T09:38:11.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:11 smithi078 ceph-mon[23694]: from='mgr.? 172.21.15.78:0/3455769200' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.rrfxge/crt"}]: dispatch 2024-09-16T09:38:11.602 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:11 smithi078 ceph-mon[23694]: from='mgr.? 172.21.15.78:0/3455769200' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-16T09:38:11.603 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:11 smithi078 ceph-mon[23694]: from='mgr.? 172.21.15.78:0/3455769200' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.rrfxge/key"}]: dispatch 2024-09-16T09:38:11.603 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:11 smithi078 ceph-mon[23694]: from='mgr.? 172.21.15.78:0/3455769200' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-16T09:38:11.603 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:11 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:11.603 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:11 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:11.738 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:11 smithi088 ceph-mon[29151]: Standby manager daemon smithi078.rrfxge restarted 2024-09-16T09:38:11.738 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:11 smithi088 ceph-mon[29151]: Standby manager daemon smithi078.rrfxge started 2024-09-16T09:38:11.738 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:11 smithi088 ceph-mon[29151]: from='mgr.? 172.21.15.78:0/3455769200' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.rrfxge/crt"}]: dispatch 2024-09-16T09:38:11.738 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:11 smithi088 ceph-mon[29151]: from='mgr.? 172.21.15.78:0/3455769200' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-16T09:38:11.738 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:11 smithi088 ceph-mon[29151]: from='mgr.? 172.21.15.78:0/3455769200' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.rrfxge/key"}]: dispatch 2024-09-16T09:38:11.738 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:11 smithi088 ceph-mon[29151]: from='mgr.? 172.21.15.78:0/3455769200' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-16T09:38:11.738 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:11 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:11.738 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:11 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:12.513 INFO:teuthology.orchestra.run.smithi078.stdout: 5 67985 11.64 MB/sec execute 32 sec latency 41.028 ms 2024-09-16T09:38:12.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:12 smithi078 ceph-mon[23694]: pgmap v7: 97 pgs: 97 active+clean; 331 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 3.0 MiB/s wr, 363 op/s 2024-09-16T09:38:12.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:12 smithi078 ceph-mon[23694]: mgrmap e27: smithi088.ncxfdz(active, since 9s), standbys: smithi078.rrfxge 2024-09-16T09:38:12.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:12 smithi088 ceph-mon[29151]: pgmap v7: 97 pgs: 97 active+clean; 331 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 3.0 MiB/s wr, 363 op/s 2024-09-16T09:38:12.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:12 smithi088 ceph-mon[29151]: mgrmap e27: smithi088.ncxfdz(active, since 9s), standbys: smithi078.rrfxge 2024-09-16T09:38:13.513 INFO:teuthology.orchestra.run.smithi078.stdout: 5 68508 11.48 MB/sec execute 33 sec latency 35.966 ms 2024-09-16T09:38:14.513 INFO:teuthology.orchestra.run.smithi078.stdout: 5 68862 11.43 MB/sec execute 34 sec latency 40.680 ms 2024-09-16T09:38:14.653 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:14 smithi078 ceph-mon[23694]: pgmap v8: 97 pgs: 97 active+clean; 331 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 2.3 MiB/s wr, 281 op/s 2024-09-16T09:38:14.653 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:14 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:14.653 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:14 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:14.654 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:14 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:14.654 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:14 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:14.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:14 smithi088 ceph-mon[29151]: pgmap v8: 97 pgs: 97 active+clean; 331 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 2.3 MiB/s wr, 281 op/s 2024-09-16T09:38:14.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:14 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:14.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:14 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:14.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:14 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:14.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:14 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:15.513 INFO:teuthology.orchestra.run.smithi078.stdout: 5 69360 11.55 MB/sec execute 35 sec latency 38.908 ms 2024-09-16T09:38:15.677 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:15 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.677 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:15 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.677 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:15 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.677 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:15 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.677 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:15 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.677 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:15 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.677 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:15 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.677 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:15 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.677 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:15 smithi078 ceph-mon[23694]: Adjusting osd_memory_target on smithi078 to 1936M 2024-09-16T09:38:15.678 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:15 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:15.678 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:15 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:15.678 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:15 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:15.678 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:15 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.678 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:15 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.678 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:15 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.678 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:15 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.678 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:15 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.678 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:15 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.678 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:15 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.678 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:15 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.678 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:15 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:15.678 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:15 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:38:15.678 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:15 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:38:15.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:15 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:15 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:15 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:15 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:15 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:15 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:15 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:15 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:15 smithi088 ceph-mon[29151]: Adjusting osd_memory_target on smithi078 to 1936M 2024-09-16T09:38:15.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:15 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:15.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:15 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:15.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:15 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:15.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:15 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:15 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:15 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:15 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:15 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:15 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:15 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:15 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-16T09:38:15.803 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:15 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:15.803 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:15 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:38:15.803 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:15 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:38:16.513 INFO:teuthology.orchestra.run.smithi078.stdout: 5 69998 11.52 MB/sec execute 36 sec latency 29.335 ms 2024-09-16T09:38:16.577 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-16T09:38:16.693 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:16 smithi078 ceph-mon[23694]: pgmap v9: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 5.1 MiB/s wr, 513 op/s 2024-09-16T09:38:16.693 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:16 smithi078 ceph-mon[23694]: Adjusting osd_memory_target on smithi088 to 2704M 2024-09-16T09:38:16.693 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:16 smithi078 ceph-mon[23694]: Updating smithi078:/etc/ceph/ceph.conf 2024-09-16T09:38:16.693 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:16 smithi078 ceph-mon[23694]: Updating smithi088:/etc/ceph/ceph.conf 2024-09-16T09:38:16.693 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:16 smithi078 ceph-mon[23694]: Updating smithi078:/var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-16T09:38:16.693 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:16 smithi078 ceph-mon[23694]: Updating smithi088:/var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-16T09:38:16.693 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:16 smithi078 ceph-mon[23694]: Updating smithi088:/etc/ceph/ceph.client.admin.keyring 2024-09-16T09:38:16.693 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:16 smithi078 ceph-mon[23694]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-09-16T09:38:16.693 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:16 smithi078 ceph-mon[23694]: Updating smithi088:/var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-16T09:38:16.693 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:16 smithi078 ceph-mon[23694]: Updating smithi078:/var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-16T09:38:16.693 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:16 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:16.694 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:16 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:16.694 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:16 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:16.694 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:16 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:16.694 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:16 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:16.694 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:16 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:16.694 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:16 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:16.694 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:16 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:16.694 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:16 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:16.694 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:16 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:17.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:16 smithi088 ceph-mon[29151]: pgmap v9: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 5.1 MiB/s wr, 513 op/s 2024-09-16T09:38:17.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:16 smithi088 ceph-mon[29151]: Adjusting osd_memory_target on smithi088 to 2704M 2024-09-16T09:38:17.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:16 smithi088 ceph-mon[29151]: Updating smithi078:/etc/ceph/ceph.conf 2024-09-16T09:38:17.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:16 smithi088 ceph-mon[29151]: Updating smithi088:/etc/ceph/ceph.conf 2024-09-16T09:38:17.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:16 smithi088 ceph-mon[29151]: Updating smithi078:/var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-16T09:38:17.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:16 smithi088 ceph-mon[29151]: Updating smithi088:/var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-16T09:38:17.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:16 smithi088 ceph-mon[29151]: Updating smithi088:/etc/ceph/ceph.client.admin.keyring 2024-09-16T09:38:17.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:16 smithi088 ceph-mon[29151]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-09-16T09:38:17.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:16 smithi088 ceph-mon[29151]: Updating smithi088:/var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-16T09:38:17.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:16 smithi088 ceph-mon[29151]: Updating smithi078:/var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-16T09:38:17.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:16 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:17.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:16 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:17.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:16 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:17.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:16 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:17.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:16 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:17.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:16 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:17.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:16 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:17.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:16 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:17.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:16 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:17.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:16 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:17.461 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T09:38:17.461 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (7m) 3s ago 10m 25.7M - 0.23.0 ba2b418f427c 08288e8d0451 2024-09-16T09:38:17.461 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (10m) 3s ago 10m 6405k - 17.2.0 e1d6a67b021e a7184b35ee05 2024-09-16T09:38:17.461 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi088 smithi088 running (8m) 7s ago 8m 6409k - 17.2.0 e1d6a67b021e a9a129cd5d7d 2024-09-16T09:38:17.461 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (7m) 3s ago 8m 54.3M - dad864ee21e9 53d95a875f02 2024-09-16T09:38:17.461 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.hiotnn smithi078 running (3m) 3s ago 3m 36.3M - 17.2.0 e1d6a67b021e c43db7767ab9 2024-09-16T09:38:17.461 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi088.dbunob smithi088 running (3m) 7s ago 3m 15.5M - 17.2.0 e1d6a67b021e d235f1bfbb05 2024-09-16T09:38:17.461 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rrfxge smithi078 *:9283 running (11m) 3s ago 11m 232M - 17.2.0 e1d6a67b021e 3445d8bfa64d 2024-09-16T09:38:17.461 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi088.ncxfdz smithi088 *:8443,9283 running (35s) 7s ago 8m 498M - 19.3.0-4882-g66e33d6a 261f2dc7b20f f9e27b9c6cd9 2024-09-16T09:38:17.461 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (11m) 3s ago 12m 54.6M 2048M 17.2.0 e1d6a67b021e c6b35fd1a57a 2024-09-16T09:38:17.461 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi088 smithi088 running (8m) 7s ago 8m 45.8M 2048M 17.2.0 e1d6a67b021e 774a14c00113 2024-09-16T09:38:17.462 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.wawosh smithi078 *:2049 running (2m) 3s ago 2m 254M - 4.0 e1d6a67b021e 2544ed9d67f5 2024-09-16T09:38:17.462 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi088.bnccmw smithi088 *:2049 running (2m) 7s ago 2m 75.6M - 4.0 e1d6a67b021e e606adb67049 2024-09-16T09:38:17.462 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (8m) 3s ago 8m 18.0M - 1.3.1 1dbe0e931976 3716a43984ae 2024-09-16T09:38:17.462 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi088 smithi088 *:9100 running (8m) 7s ago 8m 21.6M - 1.3.1 1dbe0e931976 bc997e95eb63 2024-09-16T09:38:17.462 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (7m) 3s ago 7m 299M 1936M 17.2.0 e1d6a67b021e 453389fafdff 2024-09-16T09:38:17.462 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (7m) 3s ago 7m 224M 1936M 17.2.0 e1d6a67b021e d8b72a22a11b 2024-09-16T09:38:17.462 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (6m) 3s ago 6m 174M 1936M 17.2.0 e1d6a67b021e aa521853c9bf 2024-09-16T09:38:17.462 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (6m) 3s ago 6m 241M 1936M 17.2.0 e1d6a67b021e db62172fe90c 2024-09-16T09:38:17.462 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi088 running (5m) 7s ago 5m 247M 2704M 17.2.0 e1d6a67b021e 6f4e80fe127a 2024-09-16T09:38:17.462 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi088 running (5m) 7s ago 5m 291M 2704M 17.2.0 e1d6a67b021e 73e51edb7756 2024-09-16T09:38:17.462 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi088 running (5m) 7s ago 5m 260M 2704M 17.2.0 e1d6a67b021e 7c1459b1a2d9 2024-09-16T09:38:17.462 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi088 running (4m) 7s ago 4m 358M 2704M 17.2.0 e1d6a67b021e 37f619c2ddf9 2024-09-16T09:38:17.462 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (8m) 3s ago 8m 65.3M - 2.33.4 514e6a882f6e db9476c8a1d8 2024-09-16T09:38:17.513 INFO:teuthology.orchestra.run.smithi078.stdout: 5 70386 11.62 MB/sec execute 37 sec latency 54.182 ms 2024-09-16T09:38:17.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:17 smithi078 ceph-mon[23694]: Reconfiguring alertmanager.smithi078 (dependencies changed)... 2024-09-16T09:38:17.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:17 smithi078 ceph-mon[23694]: Deploying daemon alertmanager.smithi078 on smithi078 2024-09-16T09:38:17.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:17 smithi078 ceph-mon[23694]: from='client.14774 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:38:17.998 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:38:17.998 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-16T09:38:17.998 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:38:17.999 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:38:17.999 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-16T09:38:17.999 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-16T09:38:17.999 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 1 2024-09-16T09:38:17.999 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:38:17.999 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-16T09:38:18.000 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-16T09:38:18.000 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:38:18.000 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-16T09:38:18.000 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:38:18.000 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:38:18.000 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-16T09:38:18.000 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-16T09:38:18.000 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 1 2024-09-16T09:38:18.000 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-16T09:38:18.000 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:38:18.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:17 smithi088 ceph-mon[29151]: Reconfiguring alertmanager.smithi078 (dependencies changed)... 2024-09-16T09:38:18.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:17 smithi088 ceph-mon[29151]: Deploying daemon alertmanager.smithi078 on smithi078 2024-09-16T09:38:18.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:17 smithi088 ceph-mon[29151]: from='client.14774 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:38:18.513 INFO:teuthology.orchestra.run.smithi078.stdout: 5 70856 11.78 MB/sec execute 38 sec latency 43.286 ms 2024-09-16T09:38:18.595 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:38:18.595 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f", 2024-09-16T09:38:18.595 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-16T09:38:18.596 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T09:38:18.596 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-09-16T09:38:18.596 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "1/23 daemons upgraded", 2024-09-16T09:38:18.596 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "", 2024-09-16T09:38:18.596 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-16T09:38:18.596 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:38:18.736 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:18 smithi078 ceph-mon[23694]: pgmap v10: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 4.7 MiB/s wr, 467 op/s 2024-09-16T09:38:18.737 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:18 smithi078 ceph-mon[23694]: from='client.14778 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:38:18.737 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:18 smithi078 ceph-mon[23694]: from='client.14782 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:38:18.737 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:18 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/3184330894' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T09:38:19.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:18 smithi088 ceph-mon[29151]: pgmap v10: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 4.7 MiB/s wr, 467 op/s 2024-09-16T09:38:19.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:18 smithi088 ceph-mon[29151]: from='client.14778 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:38:19.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:18 smithi088 ceph-mon[29151]: from='client.14782 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:38:19.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:18 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/3184330894' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T09:38:19.338 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-16T09:38:19.514 INFO:teuthology.orchestra.run.smithi078.stdout: 5 71391 11.96 MB/sec execute 39 sec latency 32.680 ms 2024-09-16T09:38:19.722 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:19 smithi078 ceph-mon[23694]: from='client.14790 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:38:19.722 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:19 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T09:38:19.722 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:19 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/1788886791' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T09:38:20.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:19 smithi088 ceph-mon[29151]: from='client.14790 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:38:20.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:19 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T09:38:20.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:19 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/1788886791' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T09:38:20.514 INFO:teuthology.orchestra.run.smithi078.stdout: 5 71832 11.82 MB/sec execute 40 sec latency 44.466 ms 2024-09-16T09:38:20.746 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:20 smithi078 ceph-mon[23694]: pgmap v11: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 4.7 MiB/s wr, 467 op/s 2024-09-16T09:38:20.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:20 smithi088 ceph-mon[29151]: pgmap v11: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 4.7 MiB/s wr, 467 op/s 2024-09-16T09:38:21.513 INFO:teuthology.orchestra.run.smithi078.stdout: 5 72178 11.62 MB/sec execute 41 sec latency 43.860 ms 2024-09-16T09:38:22.514 INFO:teuthology.orchestra.run.smithi078.stdout: 5 72456 11.54 MB/sec execute 42 sec latency 40.857 ms 2024-09-16T09:38:22.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:22 smithi078 ceph-mon[23694]: pgmap v12: 97 pgs: 97 active+clean; 340 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 7.7 MiB/s wr, 745 op/s 2024-09-16T09:38:23.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:22 smithi088 ceph-mon[29151]: pgmap v12: 97 pgs: 97 active+clean; 340 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 7.7 MiB/s wr, 745 op/s 2024-09-16T09:38:23.514 INFO:teuthology.orchestra.run.smithi078.stdout: 5 72939 11.62 MB/sec execute 43 sec latency 44.666 ms 2024-09-16T09:38:24.514 INFO:teuthology.orchestra.run.smithi078.stdout: 5 73395 11.50 MB/sec execute 44 sec latency 56.484 ms 2024-09-16T09:38:24.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:24 smithi088 ceph-mon[29151]: pgmap v13: 97 pgs: 97 active+clean; 340 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.0 MiB/s wr, 536 op/s 2024-09-16T09:38:24.872 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:24 smithi078 ceph-mon[23694]: pgmap v13: 97 pgs: 97 active+clean; 340 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.0 MiB/s wr, 536 op/s 2024-09-16T09:38:25.514 INFO:teuthology.orchestra.run.smithi078.stdout: 5 73767 11.51 MB/sec execute 45 sec latency 44.614 ms 2024-09-16T09:38:26.514 INFO:teuthology.orchestra.run.smithi078.stdout: 5 74022 11.46 MB/sec execute 46 sec latency 43.111 ms 2024-09-16T09:38:26.684 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:26 smithi078 ceph-mon[23694]: pgmap v14: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 793 op/s 2024-09-16T09:38:27.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:26 smithi088 ceph-mon[29151]: pgmap v14: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 793 op/s 2024-09-16T09:38:27.514 INFO:teuthology.orchestra.run.smithi078.stdout: 5 74305 11.47 MB/sec execute 47 sec latency 61.475 ms 2024-09-16T09:38:28.514 INFO:teuthology.orchestra.run.smithi078.stdout: 5 74653 11.46 MB/sec execute 48 sec latency 61.566 ms 2024-09-16T09:38:28.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:28 smithi078 ceph-mon[23694]: pgmap v15: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.2 MiB/s wr, 535 op/s 2024-09-16T09:38:29.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:28 smithi088 ceph-mon[29151]: pgmap v15: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.2 MiB/s wr, 535 op/s 2024-09-16T09:38:29.514 INFO:teuthology.orchestra.run.smithi078.stdout: 5 74894 11.43 MB/sec execute 49 sec latency 69.756 ms 2024-09-16T09:38:30.514 INFO:teuthology.orchestra.run.smithi078.stdout: 5 75045 11.27 MB/sec execute 50 sec latency 66.794 ms 2024-09-16T09:38:30.798 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:30 smithi078 ceph-mon[23694]: pgmap v16: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.2 MiB/s wr, 535 op/s 2024-09-16T09:38:30.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:30 smithi088 ceph-mon[29151]: pgmap v16: 97 pgs: 97 active+clean; 322 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.2 MiB/s wr, 535 op/s 2024-09-16T09:38:31.514 INFO:teuthology.orchestra.run.smithi078.stdout: 5 75327 11.14 MB/sec execute 51 sec latency 94.888 ms 2024-09-16T09:38:31.820 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:31 smithi078 ceph-mon[23694]: pgmap v17: 97 pgs: 97 active+clean; 329 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.5 MiB/s wr, 722 op/s 2024-09-16T09:38:32.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:31 smithi088 ceph-mon[29151]: pgmap v17: 97 pgs: 97 active+clean; 329 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.5 MiB/s wr, 722 op/s 2024-09-16T09:38:32.514 INFO:teuthology.orchestra.run.smithi078.stdout: 5 75633 10.97 MB/sec execute 52 sec latency 98.701 ms 2024-09-16T09:38:33.514 INFO:teuthology.orchestra.run.smithi078.stdout: 5 75884 10.85 MB/sec execute 53 sec latency 67.429 ms 2024-09-16T09:38:34.514 INFO:teuthology.orchestra.run.smithi078.stdout: 5 76338 10.95 MB/sec execute 54 sec latency 37.324 ms 2024-09-16T09:38:34.638 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:34 smithi078 ceph-mon[23694]: pgmap v18: 97 pgs: 97 active+clean; 329 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.5 MiB/s wr, 443 op/s 2024-09-16T09:38:34.638 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:34 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:34.638 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:34 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:34.638 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:34 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T09:38:34.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:34 smithi088 ceph-mon[29151]: pgmap v18: 97 pgs: 97 active+clean; 329 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.5 MiB/s wr, 443 op/s 2024-09-16T09:38:34.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:34 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:34.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:34 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:34.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:34 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T09:38:35.514 INFO:teuthology.orchestra.run.smithi078.stdout: 5 76936 10.94 MB/sec execute 55 sec latency 49.964 ms 2024-09-16T09:38:35.663 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:35 smithi078 ceph-mon[23694]: Reconfiguring grafana.smithi078 (dependencies changed)... 2024-09-16T09:38:35.664 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:35 smithi078 ceph-mon[23694]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-16T09:38:35.664 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:35 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:35.664 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:35 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:35.664 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:35 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-16T09:38:35.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:35 smithi088 ceph-mon[29151]: Reconfiguring grafana.smithi078 (dependencies changed)... 2024-09-16T09:38:35.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:35 smithi088 ceph-mon[29151]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-16T09:38:35.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:35 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:35.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:35 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:35.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:35 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-16T09:38:36.514 INFO:teuthology.orchestra.run.smithi078.stdout: 5 77286 10.91 MB/sec execute 56 sec latency 53.991 ms 2024-09-16T09:38:36.692 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:36 smithi078 ceph-mon[23694]: pgmap v19: 97 pgs: 97 active+clean; 320 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.1 MiB/s wr, 621 op/s 2024-09-16T09:38:36.693 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:36 smithi078 ceph-mon[23694]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-16T09:38:36.693 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:36 smithi078 ceph-mon[23694]: Reconfiguring daemon grafana.smithi078 on smithi078 2024-09-16T09:38:36.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:36 smithi088 ceph-mon[29151]: pgmap v19: 97 pgs: 97 active+clean; 320 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.1 MiB/s wr, 621 op/s 2024-09-16T09:38:36.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:36 smithi088 ceph-mon[29151]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-16T09:38:36.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:36 smithi088 ceph-mon[29151]: Reconfiguring daemon grafana.smithi078 on smithi078 2024-09-16T09:38:37.514 INFO:teuthology.orchestra.run.smithi078.stdout: 5 77536 10.92 MB/sec execute 57 sec latency 50.601 ms 2024-09-16T09:38:38.514 INFO:teuthology.orchestra.run.smithi078.stdout: 5 77791 10.90 MB/sec execute 58 sec latency 83.445 ms 2024-09-16T09:38:38.574 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:38 smithi078 ceph-mon[23694]: pgmap v20: 97 pgs: 97 active+clean; 320 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.9 MiB/s wr, 363 op/s 2024-09-16T09:38:38.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:38 smithi088 ceph-mon[29151]: pgmap v20: 97 pgs: 97 active+clean; 320 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.9 MiB/s wr, 363 op/s 2024-09-16T09:38:39.515 INFO:teuthology.orchestra.run.smithi078.stdout: 5 78099 10.91 MB/sec execute 59 sec latency 118.363 ms 2024-09-16T09:38:40.515 INFO:teuthology.orchestra.run.smithi078.stdout: 5 78363 10.91 MB/sec execute 60 sec latency 45.092 ms 2024-09-16T09:38:40.662 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:40 smithi078 ceph-mon[23694]: pgmap v21: 97 pgs: 97 active+clean; 320 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.9 MiB/s wr, 363 op/s 2024-09-16T09:38:40.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:40 smithi088 ceph-mon[29151]: pgmap v21: 97 pgs: 97 active+clean; 320 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.9 MiB/s wr, 363 op/s 2024-09-16T09:38:41.515 INFO:teuthology.orchestra.run.smithi078.stdout: 5 78621 10.81 MB/sec execute 61 sec latency 49.230 ms 2024-09-16T09:38:42.515 INFO:teuthology.orchestra.run.smithi078.stdout: 5 78933 10.71 MB/sec execute 62 sec latency 52.420 ms 2024-09-16T09:38:42.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:42 smithi078 ceph-mon[23694]: pgmap v22: 97 pgs: 97 active+clean; 327 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.2 MiB/s wr, 570 op/s 2024-09-16T09:38:42.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:42 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:42.730 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:42 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:42.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:42 smithi088 ceph-mon[29151]: pgmap v22: 97 pgs: 97 active+clean; 327 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.2 MiB/s wr, 570 op/s 2024-09-16T09:38:42.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:42 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:42.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:42 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:43.515 INFO:teuthology.orchestra.run.smithi078.stdout: 5 79248 10.59 MB/sec execute 63 sec latency 61.863 ms 2024-09-16T09:38:43.754 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:43 smithi078 ceph-mon[23694]: Reconfiguring node-exporter.smithi078 (dependencies changed)... 2024-09-16T09:38:43.755 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:43 smithi078 ceph-mon[23694]: Deploying daemon node-exporter.smithi078 on smithi078 2024-09-16T09:38:43.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:43 smithi088 ceph-mon[29151]: Reconfiguring node-exporter.smithi078 (dependencies changed)... 2024-09-16T09:38:43.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:43 smithi088 ceph-mon[29151]: Deploying daemon node-exporter.smithi078 on smithi078 2024-09-16T09:38:44.515 INFO:teuthology.orchestra.run.smithi078.stdout: 5 79534 10.53 MB/sec execute 64 sec latency 67.378 ms 2024-09-16T09:38:44.533 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:44 smithi078 ceph-mon[23694]: pgmap v23: 97 pgs: 97 active+clean; 327 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.9 MiB/s wr, 383 op/s 2024-09-16T09:38:44.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:44 smithi088 ceph-mon[29151]: pgmap v23: 97 pgs: 97 active+clean; 327 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.9 MiB/s wr, 383 op/s 2024-09-16T09:38:45.515 INFO:teuthology.orchestra.run.smithi078.stdout: 5 80249 10.68 MB/sec execute 65 sec latency 49.362 ms 2024-09-16T09:38:46.515 INFO:teuthology.orchestra.run.smithi078.stdout: 5 80707 10.65 MB/sec execute 66 sec latency 65.342 ms 2024-09-16T09:38:46.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:46 smithi078 ceph-mon[23694]: pgmap v24: 97 pgs: 97 active+clean; 324 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.7 MiB/s wr, 572 op/s 2024-09-16T09:38:46.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:46 smithi088 ceph-mon[29151]: pgmap v24: 97 pgs: 97 active+clean; 324 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.7 MiB/s wr, 572 op/s 2024-09-16T09:38:47.515 INFO:teuthology.orchestra.run.smithi078.stdout: 5 81072 10.69 MB/sec execute 67 sec latency 72.757 ms 2024-09-16T09:38:48.515 INFO:teuthology.orchestra.run.smithi078.stdout: 5 81750 10.92 MB/sec execute 68 sec latency 43.460 ms 2024-09-16T09:38:48.592 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:48 smithi078 ceph-mon[23694]: pgmap v25: 97 pgs: 97 active+clean; 324 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.1 MiB/s wr, 394 op/s 2024-09-16T09:38:48.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:48 smithi088 ceph-mon[29151]: pgmap v25: 97 pgs: 97 active+clean; 324 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.1 MiB/s wr, 394 op/s 2024-09-16T09:38:49.515 INFO:teuthology.orchestra.run.smithi078.stdout: 5 82163 10.98 MB/sec execute 69 sec latency 51.362 ms 2024-09-16T09:38:49.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:49 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T09:38:49.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:49 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T09:38:49.949 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-16T09:38:50.515 INFO:teuthology.orchestra.run.smithi078.stdout: 5 82545 10.85 MB/sec execute 70 sec latency 51.555 ms 2024-09-16T09:38:50.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:50 smithi078 ceph-mon[23694]: pgmap v26: 97 pgs: 97 active+clean; 324 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.1 MiB/s wr, 394 op/s 2024-09-16T09:38:50.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:50 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:50.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:50 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:50.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:50 smithi088 ceph-mon[29151]: pgmap v26: 97 pgs: 97 active+clean; 324 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.1 MiB/s wr, 394 op/s 2024-09-16T09:38:50.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:50 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:50.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:50 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:38:51.129 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T09:38:51.129 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 starting - - - - 2024-09-16T09:38:51.129 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (11m) 37s ago 11m 6405k - 17.2.0 e1d6a67b021e a7184b35ee05 2024-09-16T09:38:51.129 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi088 smithi088 running (9m) 40s ago 9m 6409k - 17.2.0 e1d6a67b021e a9a129cd5d7d 2024-09-16T09:38:51.129 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 starting - - - - 2024-09-16T09:38:51.129 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.hiotnn smithi078 running (3m) 37s ago 3m 36.3M - 17.2.0 e1d6a67b021e c43db7767ab9 2024-09-16T09:38:51.129 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi088.dbunob smithi088 running (3m) 40s ago 3m 15.5M - 17.2.0 e1d6a67b021e d235f1bfbb05 2024-09-16T09:38:51.129 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rrfxge smithi078 *:9283 running (12m) 37s ago 12m 232M - 17.2.0 e1d6a67b021e 3445d8bfa64d 2024-09-16T09:38:51.129 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi088.ncxfdz smithi088 *:8443,9283 running (68s) 40s ago 9m 498M - 19.3.0-4882-g66e33d6a 261f2dc7b20f f9e27b9c6cd9 2024-09-16T09:38:51.129 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (12m) 37s ago 12m 54.6M 2048M 17.2.0 e1d6a67b021e c6b35fd1a57a 2024-09-16T09:38:51.129 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi088 smithi088 running (9m) 40s ago 9m 45.8M 2048M 17.2.0 e1d6a67b021e 774a14c00113 2024-09-16T09:38:51.130 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.wawosh smithi078 *:2049 running (3m) 37s ago 3m 254M - 4.0 e1d6a67b021e 2544ed9d67f5 2024-09-16T09:38:51.130 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi088.bnccmw smithi088 *:2049 running (3m) 40s ago 3m 75.6M - 4.0 e1d6a67b021e e606adb67049 2024-09-16T09:38:51.130 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 starting - - - - 2024-09-16T09:38:51.130 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi088 smithi088 *:9100 running (8m) 40s ago 8m 21.6M - 1.3.1 1dbe0e931976 bc997e95eb63 2024-09-16T09:38:51.130 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (8m) 37s ago 8m 299M 1936M 17.2.0 e1d6a67b021e 453389fafdff 2024-09-16T09:38:51.130 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (7m) 37s ago 7m 224M 1936M 17.2.0 e1d6a67b021e d8b72a22a11b 2024-09-16T09:38:51.130 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (7m) 37s ago 7m 174M 1936M 17.2.0 e1d6a67b021e aa521853c9bf 2024-09-16T09:38:51.130 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (6m) 37s ago 6m 241M 1936M 17.2.0 e1d6a67b021e db62172fe90c 2024-09-16T09:38:51.130 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi088 running (6m) 40s ago 6m 247M 2704M 17.2.0 e1d6a67b021e 6f4e80fe127a 2024-09-16T09:38:51.130 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi088 running (6m) 40s ago 6m 291M 2704M 17.2.0 e1d6a67b021e 73e51edb7756 2024-09-16T09:38:51.130 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi088 running (5m) 40s ago 5m 260M 2704M 17.2.0 e1d6a67b021e 7c1459b1a2d9 2024-09-16T09:38:51.130 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi088 running (5m) 40s ago 5m 358M 2704M 17.2.0 e1d6a67b021e 37f619c2ddf9 2024-09-16T09:38:51.131 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (8m) 37s ago 8m 65.3M - 2.33.4 514e6a882f6e db9476c8a1d8 2024-09-16T09:38:51.515 INFO:teuthology.orchestra.run.smithi078.stdout: 5 83124 10.89 MB/sec execute 71 sec latency 20.354 ms 2024-09-16T09:38:51.636 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:51 smithi078 ceph-mon[23694]: from='client.24501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:38:51.636 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:51 smithi078 ceph-mon[23694]: Reconfiguring prometheus.smithi078 (dependencies changed)... 2024-09-16T09:38:51.637 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:51 smithi078 ceph-mon[23694]: Deploying daemon prometheus.smithi078 on smithi078 2024-09-16T09:38:51.637 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:51 smithi078 ceph-mon[23694]: from='client.14802 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:38:51.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:51 smithi088 ceph-mon[29151]: from='client.24501 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:38:51.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:51 smithi088 ceph-mon[29151]: Reconfiguring prometheus.smithi078 (dependencies changed)... 2024-09-16T09:38:51.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:51 smithi088 ceph-mon[29151]: Deploying daemon prometheus.smithi078 on smithi078 2024-09-16T09:38:51.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:51 smithi088 ceph-mon[29151]: from='client.14802 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:38:51.847 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:38:51.847 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-16T09:38:51.847 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:38:51.847 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:38:51.847 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-16T09:38:51.847 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-16T09:38:51.847 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 1 2024-09-16T09:38:51.847 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:38:51.847 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-16T09:38:51.847 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-16T09:38:51.848 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:38:51.848 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-16T09:38:51.848 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:38:51.848 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:38:51.848 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-16T09:38:51.848 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-16T09:38:51.848 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 1 2024-09-16T09:38:51.848 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-16T09:38:51.848 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:38:52.445 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:38:52.445 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f", 2024-09-16T09:38:52.445 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-16T09:38:52.445 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T09:38:52.446 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-09-16T09:38:52.446 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "1/23 daemons upgraded", 2024-09-16T09:38:52.446 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "", 2024-09-16T09:38:52.446 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-16T09:38:52.446 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:38:52.525 INFO:teuthology.orchestra.run.smithi078.stdout: 5 83630 10.96 MB/sec execute 72 sec latency 89.466 ms 2024-09-16T09:38:52.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:52 smithi078 ceph-mon[23694]: pgmap v27: 97 pgs: 97 active+clean; 335 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 7.3 MiB/s wr, 685 op/s 2024-09-16T09:38:52.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:52 smithi078 ceph-mon[23694]: from='client.14806 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:38:52.653 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:52 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/3837944860' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T09:38:52.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:52 smithi088 ceph-mon[29151]: pgmap v27: 97 pgs: 97 active+clean; 335 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 7.3 MiB/s wr, 685 op/s 2024-09-16T09:38:52.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:52 smithi088 ceph-mon[29151]: from='client.14806 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:38:52.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:52 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/3837944860' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T09:38:53.129 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-16T09:38:53.525 INFO:teuthology.orchestra.run.smithi078.stdout: 5 84151 10.91 MB/sec execute 73 sec latency 33.805 ms 2024-09-16T09:38:53.667 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:53 smithi078 ceph-mon[23694]: from='client.14812 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:38:53.667 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:53 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/2419568690' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T09:38:53.957 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:53 smithi088 ceph-mon[29151]: from='client.14812 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:38:53.957 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:53 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/2419568690' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T09:38:54.525 INFO:teuthology.orchestra.run.smithi078.stdout: 5 84548 10.91 MB/sec execute 74 sec latency 53.677 ms 2024-09-16T09:38:54.681 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:54 smithi078 ceph-mon[23694]: pgmap v28: 97 pgs: 97 active+clean; 335 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.0 MiB/s wr, 479 op/s 2024-09-16T09:38:54.949 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:54 smithi088 ceph-mon[29151]: pgmap v28: 97 pgs: 97 active+clean; 335 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.0 MiB/s wr, 479 op/s 2024-09-16T09:38:55.527 INFO:teuthology.orchestra.run.smithi078.stdout: 5 85013 11.01 MB/sec execute 75 sec latency 63.173 ms 2024-09-16T09:38:56.526 INFO:teuthology.orchestra.run.smithi078.stdout: 5 85477 11.12 MB/sec execute 76 sec latency 45.677 ms 2024-09-16T09:38:56.712 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:56 smithi078 ceph-mon[23694]: pgmap v29: 97 pgs: 97 active+clean; 322 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 764 op/s 2024-09-16T09:38:57.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:56 smithi088 ceph-mon[29151]: pgmap v29: 97 pgs: 97 active+clean; 322 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 764 op/s 2024-09-16T09:38:57.526 INFO:teuthology.orchestra.run.smithi078.stdout: 5 85764 11.08 MB/sec execute 77 sec latency 53.640 ms 2024-09-16T09:38:58.526 INFO:teuthology.orchestra.run.smithi078.stdout: 5 86039 10.97 MB/sec execute 78 sec latency 90.453 ms 2024-09-16T09:38:58.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:38:58 smithi078 ceph-mon[23694]: pgmap v30: 97 pgs: 97 active+clean; 322 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.7 MiB/s wr, 575 op/s 2024-09-16T09:38:59.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:38:58 smithi088 ceph-mon[29151]: pgmap v30: 97 pgs: 97 active+clean; 322 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.7 MiB/s wr, 575 op/s 2024-09-16T09:38:59.526 INFO:teuthology.orchestra.run.smithi078.stdout: 5 86304 10.87 MB/sec execute 79 sec latency 88.657 ms 2024-09-16T09:39:00.526 INFO:teuthology.orchestra.run.smithi078.stdout: 5 86646 10.85 MB/sec execute 80 sec latency 75.286 ms 2024-09-16T09:39:00.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:00 smithi088 ceph-mon[29151]: pgmap v31: 97 pgs: 97 active+clean; 322 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.7 MiB/s wr, 575 op/s 2024-09-16T09:39:00.802 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:00 smithi078 ceph-mon[23694]: pgmap v31: 97 pgs: 97 active+clean; 322 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.7 MiB/s wr, 575 op/s 2024-09-16T09:39:01.526 INFO:teuthology.orchestra.run.smithi078.stdout: 5 87011 10.84 MB/sec execute 81 sec latency 37.416 ms 2024-09-16T09:39:02.526 INFO:teuthology.orchestra.run.smithi078.stdout: 5 87505 10.85 MB/sec execute 82 sec latency 39.322 ms 2024-09-16T09:39:02.842 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:02 smithi078 ceph-mon[23694]: pgmap v32: 97 pgs: 97 active+clean; 333 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 794 op/s 2024-09-16T09:39:03.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:02 smithi088 ceph-mon[29151]: pgmap v32: 97 pgs: 97 active+clean; 333 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 794 op/s 2024-09-16T09:39:03.526 INFO:teuthology.orchestra.run.smithi078.stdout: 5 87902 10.81 MB/sec execute 83 sec latency 42.829 ms 2024-09-16T09:39:04.527 INFO:teuthology.orchestra.run.smithi078.stdout: 5 88239 10.82 MB/sec execute 84 sec latency 64.544 ms 2024-09-16T09:39:04.627 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:04 smithi078 ceph-mon[23694]: pgmap v33: 97 pgs: 97 active+clean; 333 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.2 MiB/s wr, 503 op/s 2024-09-16T09:39:04.628 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:04 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T09:39:04.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:04 smithi088 ceph-mon[29151]: pgmap v33: 97 pgs: 97 active+clean; 333 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.2 MiB/s wr, 503 op/s 2024-09-16T09:39:04.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:04 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T09:39:05.527 INFO:teuthology.orchestra.run.smithi078.stdout: 5 88563 10.84 MB/sec execute 85 sec latency 101.529 ms 2024-09-16T09:39:06.527 INFO:teuthology.orchestra.run.smithi078.stdout: 5 88931 10.87 MB/sec execute 86 sec latency 51.257 ms 2024-09-16T09:39:06.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:06 smithi078 ceph-mon[23694]: pgmap v34: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.3 MiB/s wr, 739 op/s 2024-09-16T09:39:07.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:06 smithi088 ceph-mon[29151]: pgmap v34: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.3 MiB/s wr, 739 op/s 2024-09-16T09:39:07.527 INFO:teuthology.orchestra.run.smithi078.stdout: 5 89316 10.90 MB/sec execute 87 sec latency 73.046 ms 2024-09-16T09:39:08.527 INFO:teuthology.orchestra.run.smithi078.stdout: 5 89498 10.80 MB/sec execute 88 sec latency 90.317 ms 2024-09-16T09:39:08.663 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:08 smithi078 ceph-mon[23694]: pgmap v35: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.7 MiB/s wr, 455 op/s 2024-09-16T09:39:09.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:08 smithi088 ceph-mon[29151]: pgmap v35: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.7 MiB/s wr, 455 op/s 2024-09-16T09:39:09.528 INFO:teuthology.orchestra.run.smithi078.stdout: 5 89675 10.70 MB/sec execute 89 sec latency 65.984 ms 2024-09-16T09:39:10.527 INFO:teuthology.orchestra.run.smithi078.stdout: 5 89915 10.61 MB/sec execute 90 sec latency 79.056 ms 2024-09-16T09:39:10.734 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:10 smithi078 ceph-mon[23694]: pgmap v36: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.7 MiB/s wr, 455 op/s 2024-09-16T09:39:11.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:10 smithi088 ceph-mon[29151]: pgmap v36: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.7 MiB/s wr, 455 op/s 2024-09-16T09:39:11.527 INFO:teuthology.orchestra.run.smithi078.stdout: 5 90121 10.58 MB/sec execute 91 sec latency 71.730 ms 2024-09-16T09:39:12.527 INFO:teuthology.orchestra.run.smithi078.stdout: 5 90300 10.51 MB/sec execute 92 sec latency 68.565 ms 2024-09-16T09:39:12.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:12 smithi088 ceph-mon[29151]: pgmap v37: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.9 MiB/s wr, 639 op/s 2024-09-16T09:39:12.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:12 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:12.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:12 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:12.858 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:12 smithi078 ceph-mon[23694]: pgmap v37: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.9 MiB/s wr, 639 op/s 2024-09-16T09:39:12.858 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:12 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:12.858 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:12 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:13.527 INFO:teuthology.orchestra.run.smithi078.stdout: 5 90630 10.54 MB/sec execute 93 sec latency 62.941 ms 2024-09-16T09:39:13.879 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:13 smithi078 ceph-mon[23694]: Reconfiguring node-exporter.smithi088 (dependencies changed)... 2024-09-16T09:39:13.879 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:13 smithi078 ceph-mon[23694]: Deploying daemon node-exporter.smithi088 on smithi088 2024-09-16T09:39:14.031 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:13 smithi088 ceph-mon[29151]: Reconfiguring node-exporter.smithi088 (dependencies changed)... 2024-09-16T09:39:14.032 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:13 smithi088 ceph-mon[29151]: Deploying daemon node-exporter.smithi088 on smithi088 2024-09-16T09:39:14.527 INFO:teuthology.orchestra.run.smithi078.stdout: 5 91100 10.50 MB/sec execute 94 sec latency 71.469 ms 2024-09-16T09:39:14.656 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:14 smithi078 ceph-mon[23694]: pgmap v38: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 420 op/s 2024-09-16T09:39:15.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:14 smithi088 ceph-mon[29151]: pgmap v38: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 420 op/s 2024-09-16T09:39:15.527 INFO:teuthology.orchestra.run.smithi078.stdout: 5 91592 10.49 MB/sec execute 95 sec latency 32.260 ms 2024-09-16T09:39:15.695 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:15 smithi078 ceph-mon[23694]: pgmap v39: 97 pgs: 97 active+clean; 322 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.6 MiB/s wr, 574 op/s 2024-09-16T09:39:16.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:15 smithi088 ceph-mon[29151]: pgmap v39: 97 pgs: 97 active+clean; 322 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.6 MiB/s wr, 574 op/s 2024-09-16T09:39:16.527 INFO:teuthology.orchestra.run.smithi078.stdout: 5 92116 10.61 MB/sec execute 96 sec latency 26.747 ms 2024-09-16T09:39:17.528 INFO:teuthology.orchestra.run.smithi078.stdout: 5 92591 10.69 MB/sec execute 97 sec latency 60.616 ms 2024-09-16T09:39:18.027 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:17 smithi078 ceph-mon[23694]: pgmap v40: 97 pgs: 97 active+clean; 322 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.5 MiB/s wr, 337 op/s 2024-09-16T09:39:18.027 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:17 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:18.027 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:17 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:18.027 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:17 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-16T09:39:18.027 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:17 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi078.front.sepia.ceph.com:9093"}]: dispatch 2024-09-16T09:39:18.027 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:17 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:18.027 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:17 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-16T09:39:18.027 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:17 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi078.front.sepia.ceph.com:3000"}]: dispatch 2024-09-16T09:39:18.027 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:17 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:18.028 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:17 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-16T09:39:18.028 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:17 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi078.front.sepia.ceph.com:9095"}]: dispatch 2024-09-16T09:39:18.028 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:17 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:18.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:17 smithi088 ceph-mon[29151]: pgmap v40: 97 pgs: 97 active+clean; 322 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.5 MiB/s wr, 337 op/s 2024-09-16T09:39:18.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:17 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:18.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:17 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:18.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:17 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-16T09:39:18.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:17 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi078.front.sepia.ceph.com:9093"}]: dispatch 2024-09-16T09:39:18.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:17 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:18.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:17 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-16T09:39:18.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:17 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi078.front.sepia.ceph.com:3000"}]: dispatch 2024-09-16T09:39:18.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:17 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:18.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:17 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-16T09:39:18.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:17 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi078.front.sepia.ceph.com:9095"}]: dispatch 2024-09-16T09:39:18.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:17 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:18.527 INFO:teuthology.orchestra.run.smithi078.stdout: 5 92999 10.67 MB/sec execute 98 sec latency 74.033 ms 2024-09-16T09:39:19.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:18 smithi078 ceph-mon[23694]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-16T09:39:19.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:18 smithi078 ceph-mon[23694]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi078.front.sepia.ceph.com:9093"}]: dispatch 2024-09-16T09:39:19.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:18 smithi078 ceph-mon[23694]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-16T09:39:19.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:18 smithi078 ceph-mon[23694]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi078.front.sepia.ceph.com:3000"}]: dispatch 2024-09-16T09:39:19.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:18 smithi078 ceph-mon[23694]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-16T09:39:19.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:18 smithi078 ceph-mon[23694]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi078.front.sepia.ceph.com:9095"}]: dispatch 2024-09-16T09:39:19.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:18 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T09:39:19.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:18 smithi078 ceph-mon[23694]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T09:39:19.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:18 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:39:19.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:18 smithi078 ceph-mon[23694]: Upgrade: Need to upgrade myself (mgr.smithi088.ncxfdz) 2024-09-16T09:39:19.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:18 smithi078 ceph-mon[23694]: Upgrade: Need to upgrade myself (mgr.smithi088.ncxfdz) 2024-09-16T09:39:19.059 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:18 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T09:39:19.059 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:18 smithi078 ceph-mon[23694]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T09:39:19.243 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:18 smithi088 ceph-mon[29151]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-16T09:39:19.243 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:18 smithi088 ceph-mon[29151]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi078.front.sepia.ceph.com:9093"}]: dispatch 2024-09-16T09:39:19.243 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:18 smithi088 ceph-mon[29151]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-16T09:39:19.243 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:18 smithi088 ceph-mon[29151]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi078.front.sepia.ceph.com:3000"}]: dispatch 2024-09-16T09:39:19.243 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:18 smithi088 ceph-mon[29151]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-16T09:39:19.243 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:18 smithi088 ceph-mon[29151]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi078.front.sepia.ceph.com:9095"}]: dispatch 2024-09-16T09:39:19.243 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:18 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T09:39:19.243 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:18 smithi088 ceph-mon[29151]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T09:39:19.244 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:18 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:39:19.244 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:18 smithi088 ceph-mon[29151]: Upgrade: Need to upgrade myself (mgr.smithi088.ncxfdz) 2024-09-16T09:39:19.244 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:18 smithi088 ceph-mon[29151]: Upgrade: Need to upgrade myself (mgr.smithi088.ncxfdz) 2024-09-16T09:39:19.244 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:18 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T09:39:19.244 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:18 smithi088 ceph-mon[29151]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T09:39:19.527 INFO:teuthology.orchestra.run.smithi078.stdout: 5 93248 10.59 MB/sec execute 99 sec latency 98.725 ms 2024-09-16T09:39:20.117 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:19 smithi078 ceph-mon[23694]: pgmap v41: 97 pgs: 97 active+clean; 322 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 3.5 MiB/s wr, 339 op/s 2024-09-16T09:39:20.117 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:19 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T09:39:20.118 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:19 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T09:39:20.118 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:19 smithi078 ceph-mon[23694]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T09:39:20.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:19 smithi088 ceph-mon[29151]: pgmap v41: 97 pgs: 97 active+clean; 322 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 3.5 MiB/s wr, 339 op/s 2024-09-16T09:39:20.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:19 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T09:39:20.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:19 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T09:39:20.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:19 smithi088 ceph-mon[29151]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T09:39:20.527 INFO:teuthology.orchestra.run.smithi078.stdout: 5 93612 10.56 MB/sec execute 100 sec latency 41.208 ms 2024-09-16T09:39:21.527 INFO:teuthology.orchestra.run.smithi078.stdout: 5 93908 10.54 MB/sec execute 101 sec latency 63.497 ms 2024-09-16T09:39:21.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:21 smithi078 ceph-mon[23694]: Upgrade: Updating mgr.smithi078.rrfxge 2024-09-16T09:39:21.653 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:21 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:21.653 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:21 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.rrfxge", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T09:39:21.653 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:21 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.rrfxge", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T09:39:21.653 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:21 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-16T09:39:21.653 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:21 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:39:21.653 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:21 smithi078 ceph-mon[23694]: Deploying daemon mgr.smithi078.rrfxge on smithi078 2024-09-16T09:39:21.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:21 smithi088 ceph-mon[29151]: Upgrade: Updating mgr.smithi078.rrfxge 2024-09-16T09:39:21.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:21 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:21.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:21 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.rrfxge", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T09:39:21.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:21 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.rrfxge", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T09:39:21.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:21 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-16T09:39:21.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:21 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:39:21.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:21 smithi088 ceph-mon[29151]: Deploying daemon mgr.smithi078.rrfxge on smithi078 2024-09-16T09:39:22.527 INFO:teuthology.orchestra.run.smithi078.stdout: 5 94353 10.58 MB/sec execute 102 sec latency 49.034 ms 2024-09-16T09:39:22.686 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:22 smithi078 ceph-mon[23694]: pgmap v42: 97 pgs: 97 active+clean; 334 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.6 MiB/s wr, 612 op/s 2024-09-16T09:39:23.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:22 smithi088 ceph-mon[29151]: pgmap v42: 97 pgs: 97 active+clean; 334 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.6 MiB/s wr, 612 op/s 2024-09-16T09:39:23.527 INFO:teuthology.orchestra.run.smithi078.stdout: 5 94732 10.51 MB/sec execute 103 sec latency 70.295 ms 2024-09-16T09:39:23.761 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-16T09:39:24.527 INFO:teuthology.orchestra.run.smithi078.stdout: 5 94984 10.46 MB/sec execute 104 sec latency 54.746 ms 2024-09-16T09:39:24.651 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T09:39:24.651 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 starting - - - - 2024-09-16T09:39:24.651 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (11m) 71s ago 11m 6405k - 17.2.0 e1d6a67b021e a7184b35ee05 2024-09-16T09:39:24.651 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi088 smithi088 running (9m) 74s ago 9m 6409k - 17.2.0 e1d6a67b021e a9a129cd5d7d 2024-09-16T09:39:24.652 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 starting - - - - 2024-09-16T09:39:24.652 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.hiotnn smithi078 running (4m) 71s ago 4m 36.3M - 17.2.0 e1d6a67b021e c43db7767ab9 2024-09-16T09:39:24.652 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi088.dbunob smithi088 running (4m) 74s ago 4m 15.5M - 17.2.0 e1d6a67b021e d235f1bfbb05 2024-09-16T09:39:24.652 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rrfxge smithi078 *:9283 running (12m) 71s ago 12m 232M - 17.2.0 e1d6a67b021e 3445d8bfa64d 2024-09-16T09:39:24.652 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi088.ncxfdz smithi088 *:8443,9283 running (102s) 74s ago 9m 498M - 19.3.0-4882-g66e33d6a 261f2dc7b20f f9e27b9c6cd9 2024-09-16T09:39:24.652 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (13m) 71s ago 13m 54.6M 2048M 17.2.0 e1d6a67b021e c6b35fd1a57a 2024-09-16T09:39:24.652 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi088 smithi088 running (9m) 74s ago 9m 45.8M 2048M 17.2.0 e1d6a67b021e 774a14c00113 2024-09-16T09:39:24.652 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.wawosh smithi078 *:2049 running (3m) 71s ago 3m 254M - 4.0 e1d6a67b021e 2544ed9d67f5 2024-09-16T09:39:24.652 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi088.bnccmw smithi088 *:2049 running (3m) 74s ago 3m 75.6M - 4.0 e1d6a67b021e e606adb67049 2024-09-16T09:39:24.652 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 starting - - - - 2024-09-16T09:39:24.652 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi088 smithi088 *:9100 starting - - - - 2024-09-16T09:39:24.652 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (8m) 71s ago 8m 299M 1936M 17.2.0 e1d6a67b021e 453389fafdff 2024-09-16T09:39:24.652 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (8m) 71s ago 8m 224M 1936M 17.2.0 e1d6a67b021e d8b72a22a11b 2024-09-16T09:39:24.653 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (7m) 71s ago 7m 174M 1936M 17.2.0 e1d6a67b021e aa521853c9bf 2024-09-16T09:39:24.653 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (7m) 71s ago 7m 241M 1936M 17.2.0 e1d6a67b021e db62172fe90c 2024-09-16T09:39:24.653 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi088 running (7m) 74s ago 7m 247M 2704M 17.2.0 e1d6a67b021e 6f4e80fe127a 2024-09-16T09:39:24.653 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi088 running (6m) 74s ago 6m 291M 2704M 17.2.0 e1d6a67b021e 73e51edb7756 2024-09-16T09:39:24.653 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi088 running (6m) 74s ago 6m 260M 2704M 17.2.0 e1d6a67b021e 7c1459b1a2d9 2024-09-16T09:39:24.653 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi088 running (5m) 74s ago 5m 358M 2704M 17.2.0 e1d6a67b021e 37f619c2ddf9 2024-09-16T09:39:24.653 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 starting - - - - 2024-09-16T09:39:24.742 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:24 smithi078 ceph-mon[23694]: pgmap v43: 97 pgs: 97 active+clean; 334 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 428 op/s 2024-09-16T09:39:24.852 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:24 smithi088 ceph-mon[29151]: pgmap v43: 97 pgs: 97 active+clean; 334 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 428 op/s 2024-09-16T09:39:25.267 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:39:25.267 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-16T09:39:25.267 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:39:25.267 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:39:25.267 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-16T09:39:25.267 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-16T09:39:25.267 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 1 2024-09-16T09:39:25.267 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:39:25.268 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-16T09:39:25.268 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-16T09:39:25.268 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:39:25.268 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-16T09:39:25.268 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:39:25.268 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:39:25.268 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-16T09:39:25.268 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-16T09:39:25.268 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 1 2024-09-16T09:39:25.268 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-16T09:39:25.268 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:39:25.527 INFO:teuthology.orchestra.run.smithi078.stdout: 5 95248 10.45 MB/sec execute 105 sec latency 84.706 ms 2024-09-16T09:39:25.784 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:25 smithi078 ceph-mon[23694]: from='client.24519 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:39:25.784 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:25 smithi078 ceph-mon[23694]: from='client.14824 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:39:25.784 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:25 smithi078 ceph-mon[23694]: from='client.24527 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:39:25.784 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:25 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/1703544982' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T09:39:25.864 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:25 smithi088 ceph-mon[29151]: from='client.24519 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:39:25.865 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:25 smithi088 ceph-mon[29151]: from='client.14824 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:39:25.865 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:25 smithi088 ceph-mon[29151]: from='client.24527 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:39:25.865 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:25 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/1703544982' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T09:39:25.911 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:39:25.911 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f", 2024-09-16T09:39:25.912 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-16T09:39:25.912 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T09:39:25.912 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-09-16T09:39:25.912 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "1/23 daemons upgraded", 2024-09-16T09:39:25.912 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading mgr daemons", 2024-09-16T09:39:25.912 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-16T09:39:25.912 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:39:26.527 INFO:teuthology.orchestra.run.smithi078.stdout: 5 95766 10.52 MB/sec execute 106 sec latency 28.063 ms 2024-09-16T09:39:26.543 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-16T09:39:26.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:26 smithi088 ceph-mon[29151]: pgmap v44: 97 pgs: 97 active+clean; 324 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.1 MiB/s wr, 643 op/s 2024-09-16T09:39:26.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:26 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/179684932' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T09:39:26.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:26 smithi078 ceph-mon[23694]: pgmap v44: 97 pgs: 97 active+clean; 324 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.1 MiB/s wr, 643 op/s 2024-09-16T09:39:26.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:26 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/179684932' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T09:39:27.527 INFO:teuthology.orchestra.run.smithi078.stdout: 5 96169 10.60 MB/sec execute 107 sec latency 50.872 ms 2024-09-16T09:39:27.816 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:27 smithi078 ceph-mon[23694]: from='client.14836 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:39:27.817 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:27 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:27.817 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:27 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:27.817 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:27 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:39:28.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:27 smithi088 ceph-mon[29151]: from='client.14836 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:39:28.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:27 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:28.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:27 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:28.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:27 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:39:28.528 INFO:teuthology.orchestra.run.smithi078.stdout: 5 96520 10.56 MB/sec execute 108 sec latency 36.103 ms 2024-09-16T09:39:28.844 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:28 smithi078 ceph-mon[23694]: pgmap v45: 97 pgs: 97 active+clean; 324 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.9 MiB/s wr, 489 op/s 2024-09-16T09:39:28.931 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:28 smithi088 ceph-mon[29151]: pgmap v45: 97 pgs: 97 active+clean; 324 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.9 MiB/s wr, 489 op/s 2024-09-16T09:39:29.528 INFO:teuthology.orchestra.run.smithi078.stdout: 5 96819 10.49 MB/sec execute 109 sec latency 79.313 ms 2024-09-16T09:39:30.528 INFO:teuthology.orchestra.run.smithi078.stdout: 5 97071 10.43 MB/sec execute 110 sec latency 51.784 ms 2024-09-16T09:39:30.635 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:30 smithi078 ceph-mon[23694]: pgmap v46: 97 pgs: 97 active+clean; 324 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 4.9 MiB/s wr, 490 op/s 2024-09-16T09:39:30.635 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:30 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:30.635 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:30 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:31.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:30 smithi088 ceph-mon[29151]: pgmap v46: 97 pgs: 97 active+clean; 324 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 4.9 MiB/s wr, 490 op/s 2024-09-16T09:39:31.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:30 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:31.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:30 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:31.528 INFO:teuthology.orchestra.run.smithi078.stdout: 5 97317 10.41 MB/sec execute 111 sec latency 54.162 ms 2024-09-16T09:39:32.528 INFO:teuthology.orchestra.run.smithi078.stdout: 5 97918 10.49 MB/sec execute 112 sec latency 26.175 ms 2024-09-16T09:39:32.658 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:32 smithi078 ceph-mon[23694]: pgmap v47: 97 pgs: 97 active+clean; 335 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 7.4 MiB/s wr, 695 op/s 2024-09-16T09:39:33.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:32 smithi088 ceph-mon[29151]: pgmap v47: 97 pgs: 97 active+clean; 335 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 7.4 MiB/s wr, 695 op/s 2024-09-16T09:39:33.528 INFO:teuthology.orchestra.run.smithi078.stdout: 5 98452 10.46 MB/sec execute 113 sec latency 30.906 ms 2024-09-16T09:39:34.528 INFO:teuthology.orchestra.run.smithi078.stdout: 5 98851 10.49 MB/sec execute 114 sec latency 24.389 ms 2024-09-16T09:39:34.672 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:34 smithi078 ceph-mon[23694]: pgmap v48: 97 pgs: 97 active+clean; 335 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.3 MiB/s wr, 422 op/s 2024-09-16T09:39:34.673 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:34 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T09:39:34.673 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:34 smithi078 ceph-mon[23694]: Standby manager daemon smithi078.rrfxge restarted 2024-09-16T09:39:34.673 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:34 smithi078 ceph-mon[23694]: Standby manager daemon smithi078.rrfxge started 2024-09-16T09:39:34.673 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:34 smithi078 ceph-mon[23694]: from='mgr.? 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.rrfxge/crt"}]: dispatch 2024-09-16T09:39:34.673 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:34 smithi078 ceph-mon[23694]: from='mgr.? 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-16T09:39:34.673 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:34 smithi078 ceph-mon[23694]: from='mgr.? 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.rrfxge/key"}]: dispatch 2024-09-16T09:39:34.673 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:34 smithi078 ceph-mon[23694]: from='mgr.? 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-16T09:39:35.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:34 smithi088 ceph-mon[29151]: pgmap v48: 97 pgs: 97 active+clean; 335 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.3 MiB/s wr, 422 op/s 2024-09-16T09:39:35.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:34 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T09:39:35.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:34 smithi088 ceph-mon[29151]: Standby manager daemon smithi078.rrfxge restarted 2024-09-16T09:39:35.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:34 smithi088 ceph-mon[29151]: Standby manager daemon smithi078.rrfxge started 2024-09-16T09:39:35.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:34 smithi088 ceph-mon[29151]: from='mgr.? 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.rrfxge/crt"}]: dispatch 2024-09-16T09:39:35.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:34 smithi088 ceph-mon[29151]: from='mgr.? 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-16T09:39:35.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:34 smithi088 ceph-mon[29151]: from='mgr.? 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.rrfxge/key"}]: dispatch 2024-09-16T09:39:35.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:34 smithi088 ceph-mon[29151]: from='mgr.? 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-16T09:39:35.528 INFO:teuthology.orchestra.run.smithi078.stdout: 5 99445 10.60 MB/sec execute 115 sec latency 32.120 ms 2024-09-16T09:39:35.684 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:35 smithi078 ceph-mon[23694]: mgrmap e28: smithi088.ncxfdz(active, since 92s), standbys: smithi078.rrfxge 2024-09-16T09:39:35.685 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:35 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:35.685 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:35 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:36.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:35 smithi088 ceph-mon[29151]: mgrmap e28: smithi088.ncxfdz(active, since 92s), standbys: smithi078.rrfxge 2024-09-16T09:39:36.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:35 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:36.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:35 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:36.528 INFO:teuthology.orchestra.run.smithi078.stdout: 5 99831 10.63 MB/sec execute 116 sec latency 50.835 ms 2024-09-16T09:39:36.697 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:36 smithi078 ceph-mon[23694]: pgmap v49: 97 pgs: 97 active+clean; 327 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.5 MiB/s wr, 667 op/s 2024-09-16T09:39:36.698 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:36 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:36.698 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:36 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:37.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:36 smithi088 ceph-mon[29151]: pgmap v49: 97 pgs: 97 active+clean; 327 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.5 MiB/s wr, 667 op/s 2024-09-16T09:39:37.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:36 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:37.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:36 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:37.528 INFO:teuthology.orchestra.run.smithi078.stdout: 5 100080 10.57 MB/sec execute 117 sec latency 71.995 ms 2024-09-16T09:39:37.702 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:37 smithi078 ceph-mon[23694]: mgrmap e29: smithi088.ncxfdz(active, since 94s), standbys: smithi078.rrfxge 2024-09-16T09:39:37.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:37 smithi088 ceph-mon[29151]: mgrmap e29: smithi088.ncxfdz(active, since 94s), standbys: smithi078.rrfxge 2024-09-16T09:39:38.528 INFO:teuthology.orchestra.run.smithi078.stdout: 5 100347 10.50 MB/sec execute 118 sec latency 47.626 ms 2024-09-16T09:39:38.718 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:38 smithi078 ceph-mon[23694]: pgmap v50: 97 pgs: 97 active+clean; 327 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 452 op/s 2024-09-16T09:39:39.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:38 smithi088 ceph-mon[29151]: pgmap v50: 97 pgs: 97 active+clean; 327 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 452 op/s 2024-09-16T09:39:39.528 INFO:teuthology.orchestra.run.smithi078.stdout: 5 100728 10.49 MB/sec execute 119 sec latency 48.043 ms 2024-09-16T09:39:40.528 INFO:teuthology.orchestra.run.smithi078.stdout: 5 101223 10.55 MB/sec execute 120 sec latency 30.622 ms 2024-09-16T09:39:40.733 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:40 smithi078 ceph-mon[23694]: pgmap v51: 97 pgs: 97 active+clean; 323 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 4.7 MiB/s wr, 454 op/s 2024-09-16T09:39:41.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:40 smithi088 ceph-mon[29151]: pgmap v51: 97 pgs: 97 active+clean; 323 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 4.7 MiB/s wr, 454 op/s 2024-09-16T09:39:41.532 INFO:teuthology.orchestra.run.smithi078.stdout: 5 101717 10.52 MB/sec execute 121 sec latency 43.291 ms 2024-09-16T09:39:42.532 INFO:teuthology.orchestra.run.smithi078.stdout: 5 102109 10.49 MB/sec execute 122 sec latency 38.831 ms 2024-09-16T09:39:42.766 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:42 smithi078 ceph-mon[23694]: pgmap v52: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 7.3 MiB/s wr, 675 op/s 2024-09-16T09:39:42.767 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:42 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:42.767 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:42 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:42.767 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:42 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:39:42.767 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:42 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:39:42.767 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:42 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:42.767 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:42 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:43.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:42 smithi088 ceph-mon[29151]: pgmap v52: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 7.3 MiB/s wr, 675 op/s 2024-09-16T09:39:43.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:42 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:43.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:42 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:43.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:42 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:39:43.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:42 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:39:43.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:42 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:43.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:42 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' 2024-09-16T09:39:43.532 INFO:teuthology.orchestra.run.smithi078.stdout: 5 102529 10.53 MB/sec execute 123 sec latency 32.148 ms 2024-09-16T09:39:43.777 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:43 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:39:43.777 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:43 smithi078 ceph-mon[23694]: Upgrade: Need to upgrade myself (mgr.smithi088.ncxfdz) 2024-09-16T09:39:43.777 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:43 smithi078 ceph-mon[23694]: Upgrade: Need to upgrade myself (mgr.smithi088.ncxfdz) 2024-09-16T09:39:43.777 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:43 smithi078 ceph-mon[23694]: Failing over to other MGR 2024-09-16T09:39:43.777 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:43 smithi078 ceph-mon[23694]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "mgr fail", "who": "smithi088.ncxfdz"}]: dispatch 2024-09-16T09:39:43.777 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:43 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "mgr fail", "who": "smithi088.ncxfdz"}]: dispatch 2024-09-16T09:39:43.777 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:43 smithi078 ceph-mon[23694]: osdmap e71: 8 total, 8 up, 8 in 2024-09-16T09:39:43.777 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:43 smithi078 ceph-mon[23694]: pgmap v54: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.7 MiB/s wr, 562 op/s 2024-09-16T09:39:44.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:43 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:39:44.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:43 smithi088 ceph-mon[29151]: Upgrade: Need to upgrade myself (mgr.smithi088.ncxfdz) 2024-09-16T09:39:44.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:43 smithi088 ceph-mon[29151]: Upgrade: Need to upgrade myself (mgr.smithi088.ncxfdz) 2024-09-16T09:39:44.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:43 smithi088 ceph-mon[29151]: Failing over to other MGR 2024-09-16T09:39:44.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:43 smithi088 ceph-mon[29151]: from='mgr.24447 172.21.15.88:0/2012125548' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "mgr fail", "who": "smithi088.ncxfdz"}]: dispatch 2024-09-16T09:39:44.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:43 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "mgr fail", "who": "smithi088.ncxfdz"}]: dispatch 2024-09-16T09:39:44.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:43 smithi088 ceph-mon[29151]: osdmap e71: 8 total, 8 up, 8 in 2024-09-16T09:39:44.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:43 smithi088 ceph-mon[29151]: pgmap v54: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.7 MiB/s wr, 562 op/s 2024-09-16T09:39:44.532 INFO:teuthology.orchestra.run.smithi078.stdout: 5 102965 10.58 MB/sec execute 124 sec latency 54.456 ms 2024-09-16T09:39:44.792 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:44 smithi078 ceph-mon[23694]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' cmd='[{"prefix": "mgr fail", "who": "smithi088.ncxfdz"}]': finished 2024-09-16T09:39:44.792 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:44 smithi078 ceph-mon[23694]: mgrmap e30: smithi078.rrfxge(active, starting, since 0.882684s) 2024-09-16T09:39:44.792 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:44 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-16T09:39:44.792 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:44 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi088"}]: dispatch 2024-09-16T09:39:44.792 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:44 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.hiotnn"}]: dispatch 2024-09-16T09:39:44.792 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:44 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi088.dbunob"}]: dispatch 2024-09-16T09:39:44.792 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:44 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mgr metadata", "who": "smithi078.rrfxge", "id": "smithi078.rrfxge"}]: dispatch 2024-09-16T09:39:44.792 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:44 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T09:39:44.792 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:44 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T09:39:44.792 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:44 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T09:39:44.793 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:44 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T09:39:44.793 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:44 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T09:39:44.793 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:44 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T09:39:44.793 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:44 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T09:39:44.793 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:44 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T09:39:44.793 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:44 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-16T09:39:44.793 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:44 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-16T09:39:44.793 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:44 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-16T09:39:44.793 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:44 smithi078 ceph-mon[23694]: Manager daemon smithi078.rrfxge is now available 2024-09-16T09:39:44.793 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:44 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:39:44.793 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:44 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T09:39:44.793 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:44 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T09:39:44.793 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:44 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:39:44.793 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:44 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/trash_purge_schedule"}]: dispatch 2024-09-16T09:39:45.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:44 smithi088 ceph-mon[29151]: from='mgr.24447 ' entity='mgr.smithi088.ncxfdz' cmd='[{"prefix": "mgr fail", "who": "smithi088.ncxfdz"}]': finished 2024-09-16T09:39:45.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:44 smithi088 ceph-mon[29151]: mgrmap e30: smithi078.rrfxge(active, starting, since 0.882684s) 2024-09-16T09:39:45.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:44 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-16T09:39:45.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:44 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi088"}]: dispatch 2024-09-16T09:39:45.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:44 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.hiotnn"}]: dispatch 2024-09-16T09:39:45.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:44 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi088.dbunob"}]: dispatch 2024-09-16T09:39:45.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:44 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mgr metadata", "who": "smithi078.rrfxge", "id": "smithi078.rrfxge"}]: dispatch 2024-09-16T09:39:45.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:44 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T09:39:45.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:44 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T09:39:45.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:44 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T09:39:45.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:44 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T09:39:45.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:44 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T09:39:45.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:44 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T09:39:45.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:44 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T09:39:45.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:44 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T09:39:45.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:44 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-16T09:39:45.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:44 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-16T09:39:45.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:44 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-16T09:39:45.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:44 smithi088 ceph-mon[29151]: Manager daemon smithi078.rrfxge is now available 2024-09-16T09:39:45.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:44 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:39:45.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:44 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T09:39:45.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:44 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T09:39:45.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:44 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:39:45.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:44 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/trash_purge_schedule"}]: dispatch 2024-09-16T09:39:45.532 INFO:teuthology.orchestra.run.smithi078.stdout: 5 103287 10.62 MB/sec execute 125 sec latency 45.137 ms 2024-09-16T09:39:45.802 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:45 smithi078 ceph-mon[23694]: mgrmap e31: smithi078.rrfxge(active, since 2s) 2024-09-16T09:39:45.802 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:45 smithi078 ceph-mon[23694]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T09:39:45.802 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:45 smithi078 ceph-mon[23694]: pgmap v3: 97 pgs: 97 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-16T09:39:45.803 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:45 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:39:45.803 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:45 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:39:46.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:45 smithi088 ceph-mon[29151]: mgrmap e31: smithi078.rrfxge(active, since 2s) 2024-09-16T09:39:46.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:45 smithi088 ceph-mon[29151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T09:39:46.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:45 smithi088 ceph-mon[29151]: pgmap v3: 97 pgs: 97 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-16T09:39:46.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:45 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:39:46.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:45 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:39:46.533 INFO:teuthology.orchestra.run.smithi078.stdout: 5 103773 10.59 MB/sec execute 126 sec latency 49.744 ms 2024-09-16T09:39:46.809 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:46 smithi078 ceph-mon[23694]: [16/Sep/2024:09:39:45] ENGINE Bus STARTING 2024-09-16T09:39:46.809 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:46 smithi078 ceph-mon[23694]: [16/Sep/2024:09:39:46] ENGINE Serving on https://172.21.15.78:7150 2024-09-16T09:39:46.809 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:46 smithi078 ceph-mon[23694]: [16/Sep/2024:09:39:46] ENGINE Error in HTTPServer.serve 2024-09-16T09:39:46.809 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Traceback (most recent call last): 2024-09-16T09:39:46.809 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-16T09:39:46.809 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self._connections.run(self.expiration_interval) 2024-09-16T09:39:46.809 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-16T09:39:46.809 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self._run(expiration_interval) 2024-09-16T09:39:46.809 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-16T09:39:46.809 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-16T09:39:46.810 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-16T09:39:46.810 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-16T09:39:46.810 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-16T09:39:46.810 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: s = self.context.wrap_socket( 2024-09-16T09:39:46.810 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-16T09:39:46.810 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: return self.sslsocket_class._create( 2024-09-16T09:39:46.810 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-16T09:39:46.810 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self.do_handshake() 2024-09-16T09:39:46.810 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-16T09:39:46.810 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self._sslobj.do_handshake() 2024-09-16T09:39:46.810 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-16T09:39:46.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:46 smithi078 ceph-mon[23694]: [16/Sep/2024:09:39:46] ENGINE Serving on http://172.21.15.78:8765 2024-09-16T09:39:46.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:46 smithi078 ceph-mon[23694]: [16/Sep/2024:09:39:46] ENGINE Bus STARTED 2024-09-16T09:39:47.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:46 smithi088 ceph-mon[29151]: [16/Sep/2024:09:39:45] ENGINE Bus STARTING 2024-09-16T09:39:47.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:46 smithi088 ceph-mon[29151]: [16/Sep/2024:09:39:46] ENGINE Serving on https://172.21.15.78:7150 2024-09-16T09:39:47.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:46 smithi088 ceph-mon[29151]: [16/Sep/2024:09:39:46] ENGINE Error in HTTPServer.serve 2024-09-16T09:39:47.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Traceback (most recent call last): 2024-09-16T09:39:47.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-16T09:39:47.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: self._connections.run(self.expiration_interval) 2024-09-16T09:39:47.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-16T09:39:47.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: self._run(expiration_interval) 2024-09-16T09:39:47.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-16T09:39:47.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-16T09:39:47.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-16T09:39:47.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-16T09:39:47.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-16T09:39:47.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: s = self.context.wrap_socket( 2024-09-16T09:39:47.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-16T09:39:47.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: return self.sslsocket_class._create( 2024-09-16T09:39:47.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-16T09:39:47.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: self.do_handshake() 2024-09-16T09:39:47.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-16T09:39:47.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: self._sslobj.do_handshake() 2024-09-16T09:39:47.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-16T09:39:47.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:46 smithi088 ceph-mon[29151]: [16/Sep/2024:09:39:46] ENGINE Serving on http://172.21.15.78:8765 2024-09-16T09:39:47.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:46 smithi088 ceph-mon[29151]: [16/Sep/2024:09:39:46] ENGINE Bus STARTED 2024-09-16T09:39:47.533 INFO:teuthology.orchestra.run.smithi078.stdout: 5 104352 10.60 MB/sec execute 127 sec latency 27.276 ms 2024-09-16T09:39:47.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:47 smithi078 ceph-mon[23694]: mgrmap e32: smithi078.rrfxge(active, since 4s) 2024-09-16T09:39:48.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:47 smithi088 ceph-mon[29151]: mgrmap e32: smithi078.rrfxge(active, since 4s) 2024-09-16T09:39:48.533 INFO:teuthology.orchestra.run.smithi078.stdout: 5 105290 10.69 MB/sec execute 128 sec latency 23.736 ms 2024-09-16T09:39:49.533 INFO:teuthology.orchestra.run.smithi078.stdout: 5 105912 10.72 MB/sec execute 129 sec latency 23.184 ms 2024-09-16T09:39:49.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:49 smithi078 ceph-mon[23694]: pgmap v5: 97 pgs: 97 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-16T09:39:49.573 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:49 smithi078 ceph-mon[23694]: mgrmap e33: smithi078.rrfxge(active, since 6s) 2024-09-16T09:39:49.573 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:49 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:39:49.573 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:49 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:39:49.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:49 smithi088 ceph-mon[29151]: pgmap v5: 97 pgs: 97 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-16T09:39:49.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:49 smithi088 ceph-mon[29151]: mgrmap e33: smithi078.rrfxge(active, since 6s) 2024-09-16T09:39:49.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:49 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:39:49.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:49 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:39:50.533 INFO:teuthology.orchestra.run.smithi078.stdout: 5 106607 10.84 MB/sec execute 130 sec latency 44.576 ms 2024-09-16T09:39:50.842 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:50 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:39:50.842 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:50 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:39:50.843 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:50 smithi078 ceph-mon[23694]: pgmap v6: 97 pgs: 97 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-16T09:39:50.843 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:50 smithi078 ceph-mon[23694]: from='mgr.? 172.21.15.88:0/691165802' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi088.ncxfdz/crt"}]: dispatch 2024-09-16T09:39:50.843 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:50 smithi078 ceph-mon[23694]: Standby manager daemon smithi088.ncxfdz started 2024-09-16T09:39:50.843 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:50 smithi078 ceph-mon[23694]: from='mgr.? 172.21.15.88:0/691165802' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-16T09:39:50.843 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:50 smithi078 ceph-mon[23694]: from='mgr.? 172.21.15.88:0/691165802' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi088.ncxfdz/key"}]: dispatch 2024-09-16T09:39:50.843 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:50 smithi078 ceph-mon[23694]: from='mgr.? 172.21.15.88:0/691165802' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-16T09:39:51.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:50 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:39:51.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:50 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:39:51.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:50 smithi088 ceph-mon[29151]: pgmap v6: 97 pgs: 97 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-16T09:39:51.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:50 smithi088 ceph-mon[29151]: from='mgr.? 172.21.15.88:0/691165802' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi088.ncxfdz/crt"}]: dispatch 2024-09-16T09:39:51.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:50 smithi088 ceph-mon[29151]: Standby manager daemon smithi088.ncxfdz started 2024-09-16T09:39:51.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:50 smithi088 ceph-mon[29151]: from='mgr.? 172.21.15.88:0/691165802' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-16T09:39:51.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:50 smithi088 ceph-mon[29151]: from='mgr.? 172.21.15.88:0/691165802' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi088.ncxfdz/key"}]: dispatch 2024-09-16T09:39:51.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:50 smithi088 ceph-mon[29151]: from='mgr.? 172.21.15.88:0/691165802' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-16T09:39:51.533 INFO:teuthology.orchestra.run.smithi078.stdout: 5 107056 10.89 MB/sec execute 131 sec latency 46.685 ms 2024-09-16T09:39:51.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:51 smithi088 ceph-mon[29151]: mgrmap e34: smithi078.rrfxge(active, since 7s), standbys: smithi088.ncxfdz 2024-09-16T09:39:51.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:51 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mgr metadata", "who": "smithi088.ncxfdz", "id": "smithi088.ncxfdz"}]: dispatch 2024-09-16T09:39:51.862 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:51 smithi078 ceph-mon[23694]: mgrmap e34: smithi078.rrfxge(active, since 7s), standbys: smithi088.ncxfdz 2024-09-16T09:39:51.862 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:51 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mgr metadata", "who": "smithi088.ncxfdz", "id": "smithi088.ncxfdz"}]: dispatch 2024-09-16T09:39:52.533 INFO:teuthology.orchestra.run.smithi078.stdout: 5 107358 10.83 MB/sec execute 132 sec latency 49.068 ms 2024-09-16T09:39:52.873 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:52 smithi078 ceph-mon[23694]: pgmap v7: 97 pgs: 97 active+clean; 344 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 6.4 MiB/s wr, 662 op/s 2024-09-16T09:39:53.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:52 smithi088 ceph-mon[29151]: pgmap v7: 97 pgs: 97 active+clean; 344 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 6.4 MiB/s wr, 662 op/s 2024-09-16T09:39:53.533 INFO:teuthology.orchestra.run.smithi078.stdout: 5 107721 10.81 MB/sec execute 133 sec latency 56.831 ms 2024-09-16T09:39:54.533 INFO:teuthology.orchestra.run.smithi078.stdout: 5 108090 10.81 MB/sec execute 134 sec latency 38.153 ms 2024-09-16T09:39:55.533 INFO:teuthology.orchestra.run.smithi078.stdout: 5 108481 10.83 MB/sec execute 135 sec latency 69.484 ms 2024-09-16T09:39:55.718 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:55 smithi078 ceph-mon[23694]: pgmap v8: 97 pgs: 97 active+clean; 344 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.9 MiB/s wr, 512 op/s 2024-09-16T09:39:55.718 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:55 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:39:55.719 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:55 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:39:55.752 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:55 smithi088 ceph-mon[29151]: pgmap v8: 97 pgs: 97 active+clean; 344 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.9 MiB/s wr, 512 op/s 2024-09-16T09:39:55.753 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:55 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:39:55.753 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:55 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:39:56.533 INFO:teuthology.orchestra.run.smithi078.stdout: 5 108870 10.77 MB/sec execute 136 sec latency 75.902 ms 2024-09-16T09:39:56.734 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:56 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:39:56.734 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:56 smithi078 ceph-mon[23694]: pgmap v9: 97 pgs: 97 active+clean; 346 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 7.1 MiB/s wr, 703 op/s 2024-09-16T09:39:56.735 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:56 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:39:57.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:56 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:39:57.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:56 smithi088 ceph-mon[29151]: pgmap v9: 97 pgs: 97 active+clean; 346 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 7.1 MiB/s wr, 703 op/s 2024-09-16T09:39:57.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:56 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:39:57.057 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-16T09:39:57.533 INFO:teuthology.orchestra.run.smithi078.stdout: 5 109139 10.73 MB/sec execute 137 sec latency 72.166 ms 2024-09-16T09:39:57.795 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:57 smithi078 ceph-mon[23694]: from='client.24567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:39:57.795 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:57 smithi078 ceph-mon[23694]: from='client.14878 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:39:57.998 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T09:39:57.999 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (85s) 3s ago 12m 17.6M - 0.27.0 11f11916f8cd 3ebb6832fbcc 2024-09-16T09:39:57.999 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (12m) 3s ago 12m 6405k - 17.2.0 e1d6a67b021e a7184b35ee05 2024-09-16T09:39:57.999 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi088 smithi088 running (10m) 9s ago 10m 6409k - 17.2.0 e1d6a67b021e a9a129cd5d7d 2024-09-16T09:39:57.999 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (77s) 3s ago 10m 44.5M - dad864ee21e9 4a91b97e29b1 2024-09-16T09:39:57.999 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.hiotnn smithi078 running (4m) 3s ago 4m 36.7M - 17.2.0 e1d6a67b021e c43db7767ab9 2024-09-16T09:39:58.000 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi088.dbunob smithi088 running (4m) 9s ago 4m 16.2M - 17.2.0 e1d6a67b021e d235f1bfbb05 2024-09-16T09:39:58.000 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rrfxge smithi078 *:8443,9283,8765 running (32s) 3s ago 13m 488M - 19.3.0-4882-g66e33d6a 261f2dc7b20f 0d25f2f30f1e 2024-09-16T09:39:58.000 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi088.ncxfdz smithi088 *:8443,9283 running (2m) 9s ago 10m 152M - 19.3.0-4882-g66e33d6a 261f2dc7b20f f9e27b9c6cd9 2024-09-16T09:39:58.000 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (13m) 3s ago 13m 60.5M 2048M 17.2.0 e1d6a67b021e c6b35fd1a57a 2024-09-16T09:39:58.000 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi088 smithi088 running (10m) 9s ago 10m 48.0M 2048M 17.2.0 e1d6a67b021e 774a14c00113 2024-09-16T09:39:58.001 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.wawosh smithi078 *:2049 running (4m) 3s ago 4m 264M - 4.0 e1d6a67b021e 2544ed9d67f5 2024-09-16T09:39:58.001 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi088.bnccmw smithi088 *:2049 running (4m) 9s ago 4m 79.8M - 4.0 e1d6a67b021e e606adb67049 2024-09-16T09:39:58.001 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (68s) 3s ago 10m 8200k - 1.7.0 72c9c2088986 895ac3ccb2f3 2024-09-16T09:39:58.001 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi088 smithi088 *:9100 running (40s) 9s ago 10m 7633k - 1.7.0 72c9c2088986 c84a2e50de53 2024-09-16T09:39:58.001 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (9m) 3s ago 9m 420M 1936M 17.2.0 e1d6a67b021e 453389fafdff 2024-09-16T09:39:58.001 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (8m) 3s ago 8m 322M 1936M 17.2.0 e1d6a67b021e d8b72a22a11b 2024-09-16T09:39:58.002 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (8m) 3s ago 8m 241M 1936M 17.2.0 e1d6a67b021e aa521853c9bf 2024-09-16T09:39:58.002 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (7m) 3s ago 7m 398M 1936M 17.2.0 e1d6a67b021e db62172fe90c 2024-09-16T09:39:58.002 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi088 running (7m) 9s ago 7m 368M 2704M 17.2.0 e1d6a67b021e 6f4e80fe127a 2024-09-16T09:39:58.002 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi088 running (7m) 9s ago 7m 399M 2704M 17.2.0 e1d6a67b021e 73e51edb7756 2024-09-16T09:39:58.002 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi088 running (6m) 9s ago 6m 335M 2704M 17.2.0 e1d6a67b021e 7c1459b1a2d9 2024-09-16T09:39:58.002 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi088 running (6m) 9s ago 6m 497M 2704M 17.2.0 e1d6a67b021e 37f619c2ddf9 2024-09-16T09:39:58.003 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (46s) 3s ago 9m 39.7M - 2.51.0 1d3b7f56885b f9f83f6add6c 2024-09-16T09:39:58.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:57 smithi088 ceph-mon[29151]: from='client.24567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:39:58.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:57 smithi088 ceph-mon[29151]: from='client.14878 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:39:58.540 INFO:teuthology.orchestra.run.smithi078.stdout: 5 109468 10.72 MB/sec execute 138 sec latency 36.978 ms 2024-09-16T09:39:58.663 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:39:58.663 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-16T09:39:58.663 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:39:58.664 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:39:58.664 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-16T09:39:58.664 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:39:58.664 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:39:58.664 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-16T09:39:58.664 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-16T09:39:58.664 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:39:58.664 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-16T09:39:58.664 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:39:58.664 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:39:58.664 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-16T09:39:58.664 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-16T09:39:58.665 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:39:58.665 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-16T09:39:58.665 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:39:59.214 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:39:59.214 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f", 2024-09-16T09:39:59.215 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-16T09:39:59.215 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T09:39:59.215 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-16T09:39:59.215 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr" 2024-09-16T09:39:59.215 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-16T09:39:59.215 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "2/23 daemons upgraded", 2024-09-16T09:39:59.215 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "", 2024-09-16T09:39:59.215 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-16T09:39:59.215 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:39:59.540 INFO:teuthology.orchestra.run.smithi078.stdout: 5 109916 10.77 MB/sec execute 139 sec latency 63.492 ms 2024-09-16T09:39:59.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:59 smithi078 ceph-mon[23694]: pgmap v10: 97 pgs: 97 active+clean; 346 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 6.4 MiB/s wr, 634 op/s 2024-09-16T09:39:59.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:59 smithi078 ceph-mon[23694]: from='client.14882 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:39:59.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:39:59 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/1663492870' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T09:39:59.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:59 smithi088 ceph-mon[29151]: pgmap v10: 97 pgs: 97 active+clean; 346 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 6.4 MiB/s wr, 634 op/s 2024-09-16T09:39:59.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:59 smithi088 ceph-mon[29151]: from='client.14882 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:39:59.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:39:59 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/1663492870' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T09:39:59.890 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-16T09:40:00.540 INFO:teuthology.orchestra.run.smithi078.stdout: 5 110412 10.85 MB/sec execute 140 sec latency 45.063 ms 2024-09-16T09:40:00.582 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:00 smithi078 ceph-mon[23694]: from='client.14890 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:40:00.582 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:00 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T09:40:00.582 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:00 smithi078 ceph-mon[23694]: from='client.? 172.21.15.78:0/2380265744' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T09:40:00.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:00 smithi078 ceph-mon[23694]: overall HEALTH_OK 2024-09-16T09:40:00.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:00 smithi088 ceph-mon[29151]: from='client.14890 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:40:00.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:00 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T09:40:00.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:00 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/2380265744' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T09:40:00.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:00 smithi088 ceph-mon[29151]: overall HEALTH_OK 2024-09-16T09:40:01.540 INFO:teuthology.orchestra.run.smithi078.stdout: 5 110747 10.81 MB/sec execute 141 sec latency 40.359 ms 2024-09-16T09:40:01.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:01 smithi078 ceph-mon[23694]: pgmap v11: 97 pgs: 97 active+clean; 346 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 6.4 MiB/s wr, 634 op/s 2024-09-16T09:40:01.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:01 smithi088 ceph-mon[29151]: pgmap v11: 97 pgs: 97 active+clean; 346 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 6.4 MiB/s wr, 634 op/s 2024-09-16T09:40:02.540 INFO:teuthology.orchestra.run.smithi078.stdout: 5 110939 10.75 MB/sec execute 142 sec latency 47.663 ms 2024-09-16T09:40:02.925 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:02 smithi078 ceph-mon[23694]: pgmap v12: 97 pgs: 97 active+clean; 344 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 8.6 MiB/s wr, 846 op/s 2024-09-16T09:40:02.925 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:02 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:02.925 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:02 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:02.925 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:02 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:40:02.925 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:02 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:40:02.925 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:02 smithi078 ceph-mon[23694]: Updating smithi078:/etc/ceph/ceph.conf 2024-09-16T09:40:02.925 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:02 smithi078 ceph-mon[23694]: Updating smithi088:/etc/ceph/ceph.conf 2024-09-16T09:40:02.925 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:02 smithi078 ceph-mon[23694]: Updating smithi088:/var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-16T09:40:02.925 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:02 smithi078 ceph-mon[23694]: Updating smithi078:/var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-16T09:40:02.925 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:02 smithi078 ceph-mon[23694]: Updating smithi088:/etc/ceph/ceph.client.admin.keyring 2024-09-16T09:40:02.926 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:02 smithi078 ceph-mon[23694]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-09-16T09:40:02.926 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:02 smithi078 ceph-mon[23694]: Updating smithi088:/var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-16T09:40:02.926 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:02 smithi078 ceph-mon[23694]: Updating smithi078:/var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-16T09:40:02.926 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:02 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:02.926 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:02 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:02.926 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:02 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:02.926 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:02 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:02.926 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:02 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:02.926 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:02 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:02.926 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:02 smithi078 ceph-mon[23694]: Reconfiguring prometheus.smithi078 (dependencies changed)... 2024-09-16T09:40:03.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:02 smithi088 ceph-mon[29151]: pgmap v12: 97 pgs: 97 active+clean; 344 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 8.6 MiB/s wr, 846 op/s 2024-09-16T09:40:03.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:02 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:03.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:02 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:03.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:02 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:40:03.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:02 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:40:03.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:02 smithi088 ceph-mon[29151]: Updating smithi078:/etc/ceph/ceph.conf 2024-09-16T09:40:03.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:02 smithi088 ceph-mon[29151]: Updating smithi088:/etc/ceph/ceph.conf 2024-09-16T09:40:03.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:02 smithi088 ceph-mon[29151]: Updating smithi088:/var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-16T09:40:03.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:02 smithi088 ceph-mon[29151]: Updating smithi078:/var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-16T09:40:03.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:02 smithi088 ceph-mon[29151]: Updating smithi088:/etc/ceph/ceph.client.admin.keyring 2024-09-16T09:40:03.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:02 smithi088 ceph-mon[29151]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-09-16T09:40:03.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:02 smithi088 ceph-mon[29151]: Updating smithi088:/var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-16T09:40:03.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:02 smithi088 ceph-mon[29151]: Updating smithi078:/var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-16T09:40:03.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:02 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:03.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:02 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:03.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:02 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:03.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:02 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:03.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:02 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:03.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:02 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:03.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:02 smithi088 ceph-mon[29151]: Reconfiguring prometheus.smithi078 (dependencies changed)... 2024-09-16T09:40:03.540 INFO:teuthology.orchestra.run.smithi078.stdout: 5 111350 10.73 MB/sec execute 143 sec latency 158.194 ms 2024-09-16T09:40:04.540 INFO:teuthology.orchestra.run.smithi078.stdout: 5 111806 10.75 MB/sec execute 144 sec latency 38.332 ms 2024-09-16T09:40:04.688 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:04 smithi078 ceph-mon[23694]: Reconfiguring daemon prometheus.smithi078 on smithi078 2024-09-16T09:40:04.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:04 smithi088 ceph-mon[29151]: Reconfiguring daemon prometheus.smithi078 on smithi078 2024-09-16T09:40:05.540 INFO:teuthology.orchestra.run.smithi078.stdout: 5 112636 10.78 MB/sec execute 145 sec latency 36.278 ms 2024-09-16T09:40:05.692 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:05 smithi078 ceph-mon[23694]: pgmap v13: 97 pgs: 97 active+clean; 344 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.0 MiB/s wr, 470 op/s 2024-09-16T09:40:05.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:05 smithi088 ceph-mon[29151]: pgmap v13: 97 pgs: 97 active+clean; 344 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.0 MiB/s wr, 470 op/s 2024-09-16T09:40:06.540 INFO:teuthology.orchestra.run.smithi078.stdout: 5 113394 10.91 MB/sec execute 146 sec latency 33.643 ms 2024-09-16T09:40:07.540 INFO:teuthology.orchestra.run.smithi078.stdout: 5 113970 10.96 MB/sec execute 147 sec latency 47.874 ms 2024-09-16T09:40:07.712 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:07 smithi078 ceph-mon[23694]: pgmap v14: 97 pgs: 97 active+clean; 346 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 713 op/s 2024-09-16T09:40:07.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:07 smithi088 ceph-mon[29151]: pgmap v14: 97 pgs: 97 active+clean; 346 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 713 op/s 2024-09-16T09:40:08.541 INFO:teuthology.orchestra.run.smithi078.stdout: 5 114426 10.95 MB/sec execute 148 sec latency 38.165 ms 2024-09-16T09:40:09.541 INFO:teuthology.orchestra.run.smithi078.stdout: 5 114961 10.95 MB/sec execute 149 sec latency 19.815 ms 2024-09-16T09:40:09.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:09 smithi078 ceph-mon[23694]: pgmap v15: 97 pgs: 97 active+clean; 346 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 454 op/s 2024-09-16T09:40:09.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:09 smithi088 ceph-mon[29151]: pgmap v15: 97 pgs: 97 active+clean; 346 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 454 op/s 2024-09-16T09:40:10.541 INFO:teuthology.orchestra.run.smithi078.stdout: 5 115408 10.97 MB/sec execute 150 sec latency 33.989 ms 2024-09-16T09:40:11.541 INFO:teuthology.orchestra.run.smithi078.stdout: 5 115766 10.96 MB/sec execute 151 sec latency 77.825 ms 2024-09-16T09:40:11.743 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:11 smithi078 ceph-mon[23694]: pgmap v16: 97 pgs: 97 active+clean; 346 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 454 op/s 2024-09-16T09:40:11.743 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:11 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:11.744 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:11 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:11.744 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:11 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-16T09:40:11.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:11 smithi088 ceph-mon[29151]: pgmap v16: 97 pgs: 97 active+clean; 346 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 454 op/s 2024-09-16T09:40:11.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:11 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:11.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:11 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:11.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:11 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-16T09:40:12.541 INFO:teuthology.orchestra.run.smithi078.stdout: 5 116114 10.91 MB/sec execute 152 sec latency 56.923 ms 2024-09-16T09:40:12.542 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:12 smithi078 ceph-mon[23694]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-16T09:40:12.542 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:12 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:40:12.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:12 smithi088 ceph-mon[29151]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-16T09:40:12.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:12 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:40:13.541 INFO:teuthology.orchestra.run.smithi078.stdout: 5 116444 10.89 MB/sec execute 153 sec latency 68.131 ms 2024-09-16T09:40:13.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:13 smithi078 ceph-mon[23694]: pgmap v17: 97 pgs: 97 active+clean; 349 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.5 MiB/s wr, 822 op/s 2024-09-16T09:40:13.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:13 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi088.ncxfdz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T09:40:13.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:13 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-16T09:40:13.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:13 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:40:13.735 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:13 smithi088 ceph-mon[29151]: pgmap v17: 97 pgs: 97 active+clean; 349 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.5 MiB/s wr, 822 op/s 2024-09-16T09:40:13.735 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:13 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi088.ncxfdz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T09:40:13.736 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:13 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-16T09:40:13.736 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:13 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:40:14.541 INFO:teuthology.orchestra.run.smithi078.stdout: 5 116913 10.96 MB/sec execute 154 sec latency 49.605 ms 2024-09-16T09:40:14.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:14 smithi078 ceph-mon[23694]: Upgrade: Updating mgr.smithi088.ncxfdz 2024-09-16T09:40:14.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:14 smithi078 ceph-mon[23694]: Deploying daemon mgr.smithi088.ncxfdz on smithi088 2024-09-16T09:40:14.793 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:14 smithi088 ceph-mon[29151]: Upgrade: Updating mgr.smithi088.ncxfdz 2024-09-16T09:40:14.793 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:14 smithi088 ceph-mon[29151]: Deploying daemon mgr.smithi088.ncxfdz on smithi088 2024-09-16T09:40:15.541 INFO:teuthology.orchestra.run.smithi078.stdout: 5 117358 10.97 MB/sec execute 155 sec latency 43.455 ms 2024-09-16T09:40:15.609 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:15 smithi078 ceph-mon[23694]: pgmap v18: 97 pgs: 97 active+clean; 349 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.2 MiB/s wr, 610 op/s 2024-09-16T09:40:15.610 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:15 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T09:40:15.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:15 smithi088 ceph-mon[29151]: pgmap v18: 97 pgs: 97 active+clean; 349 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.2 MiB/s wr, 610 op/s 2024-09-16T09:40:15.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:15 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T09:40:16.541 INFO:teuthology.orchestra.run.smithi078.stdout: 5 117800 11.00 MB/sec execute 156 sec latency 38.378 ms 2024-09-16T09:40:17.541 INFO:teuthology.orchestra.run.smithi078.stdout: 5 118056 10.94 MB/sec execute 157 sec latency 58.473 ms 2024-09-16T09:40:17.658 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:17 smithi078 ceph-mon[23694]: pgmap v19: 97 pgs: 97 active+clean; 340 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 842 op/s 2024-09-16T09:40:17.659 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:17 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:17.659 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:17 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:17.659 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:17 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:40:17.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:17 smithi088 ceph-mon[29151]: pgmap v19: 97 pgs: 97 active+clean; 340 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 842 op/s 2024-09-16T09:40:17.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:17 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:17.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:17 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:17.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:17 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:40:18.541 INFO:teuthology.orchestra.run.smithi078.stdout: 5 118388 10.92 MB/sec execute 158 sec latency 51.301 ms 2024-09-16T09:40:19.541 INFO:teuthology.orchestra.run.smithi078.stdout: 5 118662 10.89 MB/sec execute 159 sec latency 55.572 ms 2024-09-16T09:40:19.687 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:19 smithi078 ceph-mon[23694]: pgmap v20: 97 pgs: 97 active+clean; 340 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.2 MiB/s wr, 599 op/s 2024-09-16T09:40:19.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:19 smithi088 ceph-mon[29151]: pgmap v20: 97 pgs: 97 active+clean; 340 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.2 MiB/s wr, 599 op/s 2024-09-16T09:40:20.541 INFO:teuthology.orchestra.run.smithi078.stdout: 5 119002 10.89 MB/sec execute 160 sec latency 56.097 ms 2024-09-16T09:40:21.541 INFO:teuthology.orchestra.run.smithi078.stdout: 5 119413 10.88 MB/sec execute 161 sec latency 37.715 ms 2024-09-16T09:40:21.711 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:21 smithi078 ceph-mon[23694]: pgmap v21: 97 pgs: 97 active+clean; 340 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.2 MiB/s wr, 599 op/s 2024-09-16T09:40:21.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:21 smithi088 ceph-mon[29151]: pgmap v21: 97 pgs: 97 active+clean; 340 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.2 MiB/s wr, 599 op/s 2024-09-16T09:40:22.541 INFO:teuthology.orchestra.run.smithi078.stdout: 5 119887 10.86 MB/sec execute 162 sec latency 49.677 ms 2024-09-16T09:40:22.725 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:22 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:22.725 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:22 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:22.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:22 smithi078 ceph-mon[23694]: pgmap v22: 97 pgs: 97 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 810 op/s 2024-09-16T09:40:22.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:22 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:22.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:22 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:22.907 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:22 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:22.908 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:22 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:22.908 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:22 smithi088 ceph-mon[29151]: pgmap v22: 97 pgs: 97 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 810 op/s 2024-09-16T09:40:22.908 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:22 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:22.908 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:22 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:23.541 INFO:teuthology.orchestra.run.smithi078.stdout: 5 120157 10.84 MB/sec execute 163 sec latency 52.197 ms 2024-09-16T09:40:24.541 INFO:teuthology.orchestra.run.smithi078.stdout: 5 120574 10.89 MB/sec execute 164 sec latency 38.967 ms 2024-09-16T09:40:24.776 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:24 smithi078 ceph-mon[23694]: Standby manager daemon smithi088.ncxfdz restarted 2024-09-16T09:40:24.776 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:24 smithi078 ceph-mon[23694]: Standby manager daemon smithi088.ncxfdz started 2024-09-16T09:40:24.776 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:24 smithi078 ceph-mon[23694]: from='mgr.? 172.21.15.88:0/571724825' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi088.ncxfdz/crt"}]: dispatch 2024-09-16T09:40:24.776 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:24 smithi078 ceph-mon[23694]: from='mgr.? 172.21.15.88:0/571724825' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-16T09:40:24.777 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:24 smithi078 ceph-mon[23694]: from='mgr.? 172.21.15.88:0/571724825' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi088.ncxfdz/key"}]: dispatch 2024-09-16T09:40:24.777 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:24 smithi078 ceph-mon[23694]: from='mgr.? 172.21.15.88:0/571724825' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-16T09:40:24.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:24 smithi088 ceph-mon[29151]: Standby manager daemon smithi088.ncxfdz restarted 2024-09-16T09:40:24.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:24 smithi088 ceph-mon[29151]: Standby manager daemon smithi088.ncxfdz started 2024-09-16T09:40:24.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:24 smithi088 ceph-mon[29151]: from='mgr.? 172.21.15.88:0/571724825' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi088.ncxfdz/crt"}]: dispatch 2024-09-16T09:40:24.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:24 smithi088 ceph-mon[29151]: from='mgr.? 172.21.15.88:0/571724825' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-16T09:40:24.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:24 smithi088 ceph-mon[29151]: from='mgr.? 172.21.15.88:0/571724825' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi088.ncxfdz/key"}]: dispatch 2024-09-16T09:40:24.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:24 smithi088 ceph-mon[29151]: from='mgr.? 172.21.15.88:0/571724825' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-16T09:40:25.541 INFO:teuthology.orchestra.run.smithi078.stdout: 5 121022 10.92 MB/sec execute 165 sec latency 50.142 ms 2024-09-16T09:40:25.558 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:25 smithi078 ceph-mon[23694]: pgmap v23: 97 pgs: 97 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.5 MiB/s wr, 442 op/s 2024-09-16T09:40:25.558 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:25 smithi078 ceph-mon[23694]: mgrmap e35: smithi078.rrfxge(active, since 41s), standbys: smithi088.ncxfdz 2024-09-16T09:40:25.559 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:25 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:25.559 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:25 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:25.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:25 smithi088 ceph-mon[29151]: pgmap v23: 97 pgs: 97 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.5 MiB/s wr, 442 op/s 2024-09-16T09:40:25.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:25 smithi088 ceph-mon[29151]: mgrmap e35: smithi078.rrfxge(active, since 41s), standbys: smithi088.ncxfdz 2024-09-16T09:40:25.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:25 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:25.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:25 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:26.541 INFO:teuthology.orchestra.run.smithi078.stdout: 5 121339 10.91 MB/sec execute 166 sec latency 49.251 ms 2024-09-16T09:40:27.091 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:26 smithi078 ceph-mon[23694]: pgmap v24: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 678 op/s 2024-09-16T09:40:27.091 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:26 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:27.091 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:26 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:27.091 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:26 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:40:27.091 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:26 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:40:27.091 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:26 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:27.091 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:26 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:27.091 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:26 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:40:27.091 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:26 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T09:40:27.092 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:26 smithi078 ceph-mon[23694]: Upgrade: Setting container_image for all mgr 2024-09-16T09:40:27.092 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:26 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:27.092 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:26 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi078.rrfxge"}]: dispatch 2024-09-16T09:40:27.092 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:26 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi078.rrfxge"}]': finished 2024-09-16T09:40:27.092 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:26 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi088.ncxfdz"}]: dispatch 2024-09-16T09:40:27.092 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:26 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi088.ncxfdz"}]': finished 2024-09-16T09:40:27.092 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:26 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-16T09:40:27.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:26 smithi088 ceph-mon[29151]: pgmap v24: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 678 op/s 2024-09-16T09:40:27.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:26 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:27.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:26 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:27.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:26 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:40:27.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:26 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:40:27.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:26 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:27.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:26 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:27.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:26 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:40:27.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:26 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T09:40:27.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:26 smithi088 ceph-mon[29151]: Upgrade: Setting container_image for all mgr 2024-09-16T09:40:27.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:26 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:27.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:26 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi078.rrfxge"}]: dispatch 2024-09-16T09:40:27.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:26 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi078.rrfxge"}]': finished 2024-09-16T09:40:27.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:26 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi088.ncxfdz"}]: dispatch 2024-09-16T09:40:27.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:26 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi088.ncxfdz"}]': finished 2024-09-16T09:40:27.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:26 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-16T09:40:27.541 INFO:teuthology.orchestra.run.smithi078.stdout: 5 121574 10.86 MB/sec execute 167 sec latency 61.439 ms 2024-09-16T09:40:28.541 INFO:teuthology.orchestra.run.smithi078.stdout: 5 121938 10.84 MB/sec execute 168 sec latency 53.822 ms 2024-09-16T09:40:29.541 INFO:teuthology.orchestra.run.smithi078.stdout: 5 122407 10.85 MB/sec execute 169 sec latency 38.893 ms 2024-09-16T09:40:29.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:29 smithi078 ceph-mon[23694]: pgmap v25: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.6 MiB/s wr, 447 op/s 2024-09-16T09:40:29.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:29 smithi078 ceph-mon[23694]: Upgrade: Updating mon.smithi078 2024-09-16T09:40:29.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:29 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:29.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:29 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-16T09:40:29.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:29 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-16T09:40:29.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:29 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:40:29.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:29 smithi078 ceph-mon[23694]: Deploying daemon mon.smithi078 on smithi078 2024-09-16T09:40:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:29 smithi088 ceph-mon[29151]: pgmap v25: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.6 MiB/s wr, 447 op/s 2024-09-16T09:40:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:29 smithi088 ceph-mon[29151]: Upgrade: Updating mon.smithi078 2024-09-16T09:40:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:29 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:29 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-16T09:40:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:29 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-16T09:40:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:29 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:40:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:29 smithi088 ceph-mon[29151]: Deploying daemon mon.smithi078 on smithi078 2024-09-16T09:40:30.380 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-16T09:40:30.541 INFO:teuthology.orchestra.run.smithi078.stdout: 5 122780 10.86 MB/sec execute 170 sec latency 44.827 ms 2024-09-16T09:40:30.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:30 smithi078 ceph-mon[23694]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T09:40:30.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:30 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T09:40:31.172 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:31 smithi078 systemd[1]: Stopping Ceph mon.smithi078 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:40:31.260 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T09:40:31.260 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (118s) 5s ago 12m 17.6M - 0.27.0 11f11916f8cd 3ebb6832fbcc 2024-09-16T09:40:31.260 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (12m) 5s ago 12m 6405k - 17.2.0 e1d6a67b021e a7184b35ee05 2024-09-16T09:40:31.260 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi088 smithi088 running (10m) 9s ago 10m 6409k - 17.2.0 e1d6a67b021e a9a129cd5d7d 2024-09-16T09:40:31.261 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (110s) 5s ago 11m 46.4M - dad864ee21e9 4a91b97e29b1 2024-09-16T09:40:31.261 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.hiotnn smithi078 running (5m) 5s ago 5m 36.3M - 17.2.0 e1d6a67b021e c43db7767ab9 2024-09-16T09:40:31.261 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi088.dbunob smithi088 running (5m) 9s ago 5m 16.4M - 17.2.0 e1d6a67b021e d235f1bfbb05 2024-09-16T09:40:31.261 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rrfxge smithi078 *:8443,9283,8765 running (65s) 5s ago 14m 506M - 19.3.0-4882-g66e33d6a 261f2dc7b20f 0d25f2f30f1e 2024-09-16T09:40:31.261 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi088.ncxfdz smithi088 *:8443,9283,8765 running (14s) 9s ago 10m 64.4M - 19.3.0-4882-g66e33d6a 261f2dc7b20f e2d03f701acd 2024-09-16T09:40:31.261 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (14m) 5s ago 14m 60.1M 2048M 17.2.0 e1d6a67b021e c6b35fd1a57a 2024-09-16T09:40:31.261 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi088 smithi088 running (10m) 9s ago 10m 48.2M 2048M 17.2.0 e1d6a67b021e 774a14c00113 2024-09-16T09:40:31.261 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.wawosh smithi078 *:2049 running (5m) 5s ago 5m 273M - 4.0 e1d6a67b021e 2544ed9d67f5 2024-09-16T09:40:31.261 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi088.bnccmw smithi088 *:2049 running (4m) 9s ago 4m 81.3M - 4.0 e1d6a67b021e e606adb67049 2024-09-16T09:40:31.261 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (102s) 5s ago 11m 8178k - 1.7.0 72c9c2088986 895ac3ccb2f3 2024-09-16T09:40:31.261 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi088 smithi088 *:9100 running (74s) 9s ago 10m 9.83M - 1.7.0 72c9c2088986 c84a2e50de53 2024-09-16T09:40:31.261 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (9m) 5s ago 9m 455M 1936M 17.2.0 e1d6a67b021e 453389fafdff 2024-09-16T09:40:31.261 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (9m) 5s ago 9m 389M 1936M 17.2.0 e1d6a67b021e d8b72a22a11b 2024-09-16T09:40:31.261 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (8m) 5s ago 8m 270M 1936M 17.2.0 e1d6a67b021e aa521853c9bf 2024-09-16T09:40:31.262 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (8m) 5s ago 8m 419M 1936M 17.2.0 e1d6a67b021e db62172fe90c 2024-09-16T09:40:31.262 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi088 running (8m) 9s ago 8m 407M 2704M 17.2.0 e1d6a67b021e 6f4e80fe127a 2024-09-16T09:40:31.262 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi088 running (7m) 9s ago 7m 454M 2704M 17.2.0 e1d6a67b021e 73e51edb7756 2024-09-16T09:40:31.262 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi088 running (7m) 9s ago 7m 403M 2704M 17.2.0 e1d6a67b021e 7c1459b1a2d9 2024-09-16T09:40:31.262 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi088 running (6m) 9s ago 6m 535M 2704M 17.2.0 e1d6a67b021e 37f619c2ddf9 2024-09-16T09:40:31.262 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (20s) 5s ago 10m 43.7M - 2.51.0 1d3b7f56885b 4c956ead9125 2024-09-16T09:40:31.541 INFO:teuthology.orchestra.run.smithi078.stdout: 5 123118 10.81 MB/sec execute 171 sec latency 65.541 ms 2024-09-16T09:40:31.673 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:31 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi078[23690]: 2024-09-16T09:40:31.503+0000 7f460a260700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi078 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T09:40:31.673 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:31 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi078[23690]: 2024-09-16T09:40:31.503+0000 7f460a260700 -1 mon.smithi078@0(leader) e2 *** Got Signal Terminated *** 2024-09-16T09:40:31.673 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:31 smithi078 podman[63676]: 2024-09-16 09:40:31.565449318 +0000 UTC m=+0.232556789 container died c6b35fd1a57a7f7b09fbe05908c1d74b69ebc365f3970d351dc1b0f427235b81 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi078, maintainer=Guillaume Abrioux , ceph=True, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, GIT_CLEAN=True, release=754, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-16T09:40:31.828 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:40:31.829 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-16T09:40:31.829 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:40:31.829 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:40:31.829 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-16T09:40:31.829 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:40:31.829 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:40:31.829 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-16T09:40:31.829 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-16T09:40:31.829 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:40:31.829 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-16T09:40:31.829 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:40:31.830 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:40:31.830 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-16T09:40:31.830 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-16T09:40:31.830 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:40:31.830 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-16T09:40:31.830 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:40:32.278 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:40:32.278 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f", 2024-09-16T09:40:32.278 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-16T09:40:32.278 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T09:40:32.278 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-16T09:40:32.278 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr" 2024-09-16T09:40:32.278 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-16T09:40:32.278 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "2/23 daemons upgraded", 2024-09-16T09:40:32.278 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading mon daemons", 2024-09-16T09:40:32.278 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-16T09:40:32.278 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:40:32.440 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:32 smithi078 podman[63676]: 2024-09-16 09:40:32.241754244 +0000 UTC m=+0.908861711 container cleanup c6b35fd1a57a7f7b09fbe05908c1d74b69ebc365f3970d351dc1b0f427235b81 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi078, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, version=8, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, release=754, ceph=True, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-16T09:40:32.440 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:32 smithi078 bash[63676]: ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi078 2024-09-16T09:40:32.541 INFO:teuthology.orchestra.run.smithi078.stdout: 5 123498 10.80 MB/sec execute 172 sec latency 38.071 ms 2024-09-16T09:40:32.691 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:32 smithi078 podman[63707]: 2024-09-16 09:40:32.652808306 +0000 UTC m=+1.089478148 container remove c6b35fd1a57a7f7b09fbe05908c1d74b69ebc365f3970d351dc1b0f427235b81 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi078, vcs-type=git, distribution-scope=public, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, name=centos-stream, ceph=True, maintainer=Guillaume Abrioux , release=754) 2024-09-16T09:40:32.838 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-16T09:40:33.212 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:32 smithi078 systemd[1]: ceph-a70c765e-740d-11ef-bceb-c7b262605968@mon.smithi078.service: Deactivated successfully. 2024-09-16T09:40:33.213 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:32 smithi078 systemd[1]: Stopped Ceph mon.smithi078 for a70c765e-740d-11ef-bceb-c7b262605968. 2024-09-16T09:40:33.213 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:32 smithi078 systemd[1]: ceph-a70c765e-740d-11ef-bceb-c7b262605968@mon.smithi078.service: Consumed 16.324s CPU time. 2024-09-16T09:40:33.464 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:33 smithi078 systemd[1]: Starting Ceph mon.smithi078 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:40:33.542 INFO:teuthology.orchestra.run.smithi078.stdout: 5 123722 10.77 MB/sec execute 173 sec latency 71.276 ms 2024-09-16T09:40:33.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:33 smithi078 podman[63891]: 2024-09-16 09:40:33.501945367 +0000 UTC m=+0.025501695 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:40:33.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:33 smithi078 podman[63891]: 2024-09-16 09:40:33.637211287 +0000 UTC m=+0.160767595 container create f5a3a0fec7570aa41f61afb64ca6a348ab03e2bc531672b34a47a0135b944c31 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi078, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-16T09:40:34.251 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 podman[63891]: 2024-09-16 09:40:34.015080383 +0000 UTC m=+0.538636694 container init f5a3a0fec7570aa41f61afb64ca6a348ab03e2bc531672b34a47a0135b944c31 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi078, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-16T09:40:34.252 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 podman[63891]: 2024-09-16 09:40:34.028995012 +0000 UTC m=+0.552551323 container start f5a3a0fec7570aa41f61afb64ca6a348ab03e2bc531672b34a47a0135b944c31 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi078, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-16T09:40:34.252 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: set uid:gid to 167:167 (ceph:ceph) 2024-09-16T09:40:34.252 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev), process ceph-mon, pid 2 2024-09-16T09:40:34.252 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: pidfile_write: ignore empty --pid-file 2024-09-16T09:40:34.252 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: load: jerasure load: lrc 2024-09-16T09:40:34.252 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: RocksDB version: 7.9.2 2024-09-16T09:40:34.252 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Git sha 0 2024-09-16T09:40:34.252 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Compile date 2024-09-12 18:19:39 2024-09-16T09:40:34.252 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: DB SUMMARY 2024-09-16T09:40:34.252 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: DB Session ID: SWBL7GYMTPVQDG751I7S 2024-09-16T09:40:34.252 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: CURRENT file: CURRENT 2024-09-16T09:40:34.252 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: IDENTITY file: IDENTITY 2024-09-16T09:40:34.252 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: MANIFEST file: MANIFEST-000015 size: 957 Bytes 2024-09-16T09:40:34.253 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi078/store.db dir, Total Num: 1, files: 000030.sst 2024-09-16T09:40:34.253 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi078/store.db: 000028.log size: 4038829 ; 2024-09-16T09:40:34.253 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.error_if_exists: 0 2024-09-16T09:40:34.253 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.create_if_missing: 0 2024-09-16T09:40:34.253 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.paranoid_checks: 1 2024-09-16T09:40:34.253 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-16T09:40:34.253 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-16T09:40:34.253 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-16T09:40:34.253 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.env: 0x56058ccfb120 2024-09-16T09:40:34.253 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.fs: PosixFileSystem 2024-09-16T09:40:34.253 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.info_log: 0x56058d83b8e0 2024-09-16T09:40:34.253 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.max_file_opening_threads: 16 2024-09-16T09:40:34.253 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.statistics: (nil) 2024-09-16T09:40:34.254 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.use_fsync: 0 2024-09-16T09:40:34.254 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.max_log_file_size: 0 2024-09-16T09:40:34.254 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-16T09:40:34.254 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-16T09:40:34.254 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.keep_log_file_num: 1000 2024-09-16T09:40:34.254 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.recycle_log_file_num: 0 2024-09-16T09:40:34.254 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.allow_fallocate: 1 2024-09-16T09:40:34.254 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.allow_mmap_reads: 0 2024-09-16T09:40:34.254 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.allow_mmap_writes: 0 2024-09-16T09:40:34.254 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.use_direct_reads: 0 2024-09-16T09:40:34.254 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-16T09:40:34.254 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.create_missing_column_families: 0 2024-09-16T09:40:34.254 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.db_log_dir: 2024-09-16T09:40:34.255 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.wal_dir: 2024-09-16T09:40:34.255 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-16T09:40:34.255 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-16T09:40:34.255 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-16T09:40:34.255 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-16T09:40:34.255 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-16T09:40:34.255 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-16T09:40:34.255 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.advise_random_on_open: 1 2024-09-16T09:40:34.255 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.db_write_buffer_size: 0 2024-09-16T09:40:34.255 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.write_buffer_manager: 0x56058d83fb80 2024-09-16T09:40:34.255 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-16T09:40:34.255 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-16T09:40:34.255 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-16T09:40:34.255 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.rate_limiter: (nil) 2024-09-16T09:40:34.256 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-16T09:40:34.256 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.wal_recovery_mode: 2 2024-09-16T09:40:34.256 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.enable_thread_tracking: 0 2024-09-16T09:40:34.256 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.enable_pipelined_write: 0 2024-09-16T09:40:34.256 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.unordered_write: 0 2024-09-16T09:40:34.256 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-16T09:40:34.256 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-16T09:40:34.256 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-16T09:40:34.256 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-16T09:40:34.256 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.row_cache: None 2024-09-16T09:40:34.256 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.wal_filter: None 2024-09-16T09:40:34.257 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-16T09:40:34.257 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.allow_ingest_behind: 0 2024-09-16T09:40:34.258 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.two_write_queues: 0 2024-09-16T09:40:34.258 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.manual_wal_flush: 0 2024-09-16T09:40:34.258 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.wal_compression: 0 2024-09-16T09:40:34.258 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.atomic_flush: 0 2024-09-16T09:40:34.258 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-16T09:40:34.258 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-16T09:40:34.258 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-16T09:40:34.258 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.log_readahead_size: 0 2024-09-16T09:40:34.258 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-16T09:40:34.258 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.best_efforts_recovery: 0 2024-09-16T09:40:34.258 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-16T09:40:34.258 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-16T09:40:34.258 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.allow_data_in_errors: 0 2024-09-16T09:40:34.258 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.db_host_id: __hostname__ 2024-09-16T09:40:34.259 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-16T09:40:34.259 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.max_background_jobs: 2 2024-09-16T09:40:34.259 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.max_background_compactions: -1 2024-09-16T09:40:34.259 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.max_subcompactions: 1 2024-09-16T09:40:34.259 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-16T09:40:34.259 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-16T09:40:34.259 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-16T09:40:34.259 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.max_total_wal_size: 0 2024-09-16T09:40:34.259 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-16T09:40:34.259 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-16T09:40:34.259 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-16T09:40:34.259 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-16T09:40:34.259 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.max_open_files: -1 2024-09-16T09:40:34.259 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.bytes_per_sync: 0 2024-09-16T09:40:34.260 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-16T09:40:34.260 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-16T09:40:34.260 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.compaction_readahead_size: 0 2024-09-16T09:40:34.260 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.max_background_flushes: -1 2024-09-16T09:40:34.260 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Compression algorithms supported: 2024-09-16T09:40:34.260 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: kZSTD supported: 0 2024-09-16T09:40:34.260 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: kXpressCompression supported: 0 2024-09-16T09:40:34.260 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: kBZip2Compression supported: 0 2024-09-16T09:40:34.260 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-16T09:40:34.260 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: kLZ4Compression supported: 1 2024-09-16T09:40:34.260 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: kZlibCompression supported: 1 2024-09-16T09:40:34.260 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: kLZ4HCCompression supported: 1 2024-09-16T09:40:34.260 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: kSnappyCompression supported: 1 2024-09-16T09:40:34.260 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-16T09:40:34.261 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-16T09:40:34.261 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi078/store.db/MANIFEST-000015 2024-09-16T09:40:34.261 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-16T09:40:34.261 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-16T09:40:34.261 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.merge_operator: 2024-09-16T09:40:34.261 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.compaction_filter: None 2024-09-16T09:40:34.261 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.compaction_filter_factory: None 2024-09-16T09:40:34.261 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.sst_partitioner_factory: None 2024-09-16T09:40:34.261 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-16T09:40:34.261 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-16T09:40:34.261 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56058d83b540) 2024-09-16T09:40:34.261 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: cache_index_and_filter_blocks: 1 2024-09-16T09:40:34.261 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-16T09:40:34.262 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-16T09:40:34.262 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: pin_top_level_index_and_filter: 1 2024-09-16T09:40:34.262 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: index_type: 0 2024-09-16T09:40:34.262 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: data_block_index_type: 0 2024-09-16T09:40:34.262 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: index_shortening: 1 2024-09-16T09:40:34.262 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-16T09:40:34.262 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: checksum: 4 2024-09-16T09:40:34.262 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: no_block_cache: 0 2024-09-16T09:40:34.262 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache: 0x56058d82f610 2024-09-16T09:40:34.262 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache_name: BinnedLRUCache 2024-09-16T09:40:34.262 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache_options: 2024-09-16T09:40:34.262 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: capacity : 536870912 2024-09-16T09:40:34.262 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: num_shard_bits : 4 2024-09-16T09:40:34.262 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: strict_capacity_limit : 0 2024-09-16T09:40:34.262 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: high_pri_pool_ratio: 0.000 2024-09-16T09:40:34.263 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache_compressed: (nil) 2024-09-16T09:40:34.263 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: persistent_cache: (nil) 2024-09-16T09:40:34.263 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_size: 4096 2024-09-16T09:40:34.263 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_size_deviation: 10 2024-09-16T09:40:34.263 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_restart_interval: 16 2024-09-16T09:40:34.263 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: index_block_restart_interval: 1 2024-09-16T09:40:34.263 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: metadata_block_size: 4096 2024-09-16T09:40:34.263 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: partition_filters: 0 2024-09-16T09:40:34.263 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: use_delta_encoding: 1 2024-09-16T09:40:34.263 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: filter_policy: bloomfilter 2024-09-16T09:40:34.263 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: whole_key_filtering: 1 2024-09-16T09:40:34.263 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: verify_compression: 0 2024-09-16T09:40:34.263 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: read_amp_bytes_per_bit: 0 2024-09-16T09:40:34.263 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: format_version: 5 2024-09-16T09:40:34.264 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: enable_index_compression: 1 2024-09-16T09:40:34.264 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_align: 0 2024-09-16T09:40:34.264 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: max_auto_readahead_size: 262144 2024-09-16T09:40:34.264 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: prepopulate_block_cache: 0 2024-09-16T09:40:34.264 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: initial_auto_readahead_size: 8192 2024-09-16T09:40:34.264 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: num_file_reads_for_auto_readahead: 2 2024-09-16T09:40:34.264 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.write_buffer_size: 33554432 2024-09-16T09:40:34.264 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.max_write_buffer_number: 2 2024-09-16T09:40:34.264 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.compression: NoCompression 2024-09-16T09:40:34.264 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.bottommost_compression: Disabled 2024-09-16T09:40:34.264 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.prefix_extractor: nullptr 2024-09-16T09:40:34.264 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-16T09:40:34.264 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.num_levels: 7 2024-09-16T09:40:34.264 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-16T09:40:34.265 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-16T09:40:34.265 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-16T09:40:34.265 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-16T09:40:34.265 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-16T09:40:34.265 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-16T09:40:34.265 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-16T09:40:34.265 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-16T09:40:34.265 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-16T09:40:34.265 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-16T09:40:34.265 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-16T09:40:34.265 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-16T09:40:34.265 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-16T09:40:34.265 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.compression_opts.level: 32767 2024-09-16T09:40:34.266 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.compression_opts.strategy: 0 2024-09-16T09:40:34.266 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-16T09:40:34.266 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-16T09:40:34.266 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-16T09:40:34.266 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-16T09:40:34.266 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.compression_opts.enabled: false 2024-09-16T09:40:34.266 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-16T09:40:34.266 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-16T09:40:34.266 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-16T09:40:34.266 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-16T09:40:34.266 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.target_file_size_base: 67108864 2024-09-16T09:40:34.266 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-16T09:40:34.267 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-16T09:40:34.267 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-16T09:40:34.267 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-16T09:40:34.267 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-16T09:40:34.267 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-16T09:40:34.267 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-16T09:40:34.267 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-16T09:40:34.267 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-16T09:40:34.267 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-16T09:40:34.267 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-16T09:40:34.267 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-16T09:40:34.267 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-16T09:40:34.267 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-16T09:40:34.267 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.arena_block_size: 1048576 2024-09-16T09:40:34.267 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-16T09:40:34.267 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-16T09:40:34.268 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.disable_auto_compactions: 0 2024-09-16T09:40:34.268 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-16T09:40:34.268 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-16T09:40:34.268 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-16T09:40:34.268 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-16T09:40:34.268 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-16T09:40:34.268 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-16T09:40:34.268 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-16T09:40:34.268 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-16T09:40:34.268 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-16T09:40:34.268 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-16T09:40:34.268 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-16T09:40:34.268 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.inplace_update_support: 0 2024-09-16T09:40:34.268 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-16T09:40:34.269 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-16T09:40:34.269 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-16T09:40:34.269 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-16T09:40:34.269 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.bloom_locality: 0 2024-09-16T09:40:34.269 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.max_successive_merges: 0 2024-09-16T09:40:34.269 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-16T09:40:34.269 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.paranoid_file_checks: 0 2024-09-16T09:40:34.269 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.force_consistency_checks: 1 2024-09-16T09:40:34.269 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.report_bg_io_stats: 0 2024-09-16T09:40:34.269 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.ttl: 2592000 2024-09-16T09:40:34.269 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-16T09:40:34.270 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-16T09:40:34.270 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-16T09:40:34.270 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.enable_blob_files: false 2024-09-16T09:40:34.270 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.min_blob_size: 0 2024-09-16T09:40:34.270 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.blob_file_size: 268435456 2024-09-16T09:40:34.270 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-16T09:40:34.270 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-16T09:40:34.270 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-16T09:40:34.270 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-16T09:40:34.270 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-16T09:40:34.270 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.blob_file_starting_level: 0 2024-09-16T09:40:34.270 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-16T09:40:34.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-09-16T09:40:34.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-16T09:40:34.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi078/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 11755, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-16T09:40:34.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-09-16T09:40:34.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 1588589c-91b7-4c11-913a-ee25adeea195 2024-09-16T09:40:34.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726479634066472, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-09-16T09:40:34.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-09-16T09:40:34.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726479634084185, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 3693275, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11742, "largest_seqno": 14165, "table_properties": {"data_size": 3683004, "index_size": 6344, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 2885, "raw_key_size": 24657, "raw_average_key_size": 21, "raw_value_size": 3660557, "raw_average_value_size": 3228, "num_data_blocks": 288, "num_entries": 1134, "num_filter_entries": 1134, "num_deletions": 253, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726479634, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "1588589c-91b7-4c11-913a-ee25adeea195", "db_session_id": "SWBL7GYMTPVQDG751I7S", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-09-16T09:40:34.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726479634084325, "job": 1, "event": "recovery_finished"} 2024-09-16T09:40:34.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-09-16T09:40:34.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-16T09:40:34.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi078/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-16T09:40:34.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x56058d85ee00 2024-09-16T09:40:34.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: DB pointer 0x56058d960000 2024-09-16T09:40:34.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-16T09:40:34.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-16T09:40:34.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** DB Stats ** 2024-09-16T09:40:34.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-16T09:40:34.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-16T09:40:34.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-16T09:40:34.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-16T09:40:34.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-16T09:40:34.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-16T09:40:34.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-16T09:40:34.273 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-16T09:40:34.273 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** Compaction Stats [default] ** 2024-09-16T09:40:34.273 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-16T09:40:34.273 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-16T09:40:34.273 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: L0 1/0 3.52 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 393.4 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-16T09:40:34.273 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: L6 1/0 6.40 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-16T09:40:34.273 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Sum 2/0 9.92 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 393.4 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-16T09:40:34.273 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 393.4 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-16T09:40:34.273 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-16T09:40:34.273 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** Compaction Stats [default] ** 2024-09-16T09:40:34.273 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-16T09:40:34.273 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-16T09:40:34.274 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 393.4 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-16T09:40:34.274 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-16T09:40:34.274 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-16T09:40:34.274 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-16T09:40:34.274 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-16T09:40:34.274 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Flush(GB): cumulative 0.003, interval 0.003 2024-09-16T09:40:34.274 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-16T09:40:34.274 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-16T09:40:34.274 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-16T09:40:34.274 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-16T09:40:34.274 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative compaction: 0.00 GB write, 157.30 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-16T09:40:34.274 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval compaction: 0.00 GB write, 157.30 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-16T09:40:34.275 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-16T09:40:34.275 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Block cache BinnedLRUCache@0x56058d82f610#2 capacity: 512.00 MB usage: 9.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.5e-05 secs_since: 0 2024-09-16T09:40:34.275 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,3.03 KB,0.000578165%) IndexBlock(1,6.58 KB,0.00125468%) Misc(1,0.00 KB,0%) 2024-09-16T09:40:34.275 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-16T09:40:34.275 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-16T09:40:34.275 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: starting mon.smithi078 rank 0 at public addrs [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] at bind addrs [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi078 fsid a70c765e-740d-11ef-bceb-c7b262605968 2024-09-16T09:40:34.275 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: mon.smithi078@-1(???) e2 preinit fsid a70c765e-740d-11ef-bceb-c7b262605968 2024-09-16T09:40:34.275 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: mon.smithi078@-1(???).mds e7 new map 2024-09-16T09:40:34.275 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: mon.smithi078@-1(???).mds e7 print_map 2024-09-16T09:40:34.275 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: e7 2024-09-16T09:40:34.275 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-16T09:40:34.275 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-16T09:40:34.275 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-16T09:40:34.276 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: legacy client fscid: 1 2024-09-16T09:40:34.276 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-16T09:40:34.276 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Filesystem 'foofs' (1) 2024-09-16T09:40:34.276 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: fs_name foofs 2024-09-16T09:40:34.276 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: epoch 5 2024-09-16T09:40:34.276 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-16T09:40:34.276 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: created 2024-09-16T09:35:01.623131+0000 2024-09-16T09:40:34.276 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: modified 2024-09-16T09:35:07.623249+0000 2024-09-16T09:40:34.276 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: tableserver 0 2024-09-16T09:40:34.276 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: root 0 2024-09-16T09:40:34.276 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: session_timeout 60 2024-09-16T09:40:34.276 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: session_autoclose 300 2024-09-16T09:40:34.277 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: max_file_size 1099511627776 2024-09-16T09:40:34.277 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: max_xattr_size 65536 2024-09-16T09:40:34.277 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: required_client_features {} 2024-09-16T09:40:34.277 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: last_failure 0 2024-09-16T09:40:34.277 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: last_failure_osd_epoch 0 2024-09-16T09:40:34.277 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-16T09:40:34.277 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: max_mds 1 2024-09-16T09:40:34.277 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: in 0 2024-09-16T09:40:34.277 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: up {0=14538} 2024-09-16T09:40:34.277 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: failed 2024-09-16T09:40:34.277 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: damaged 2024-09-16T09:40:34.277 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: stopped 2024-09-16T09:40:34.277 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: data_pools [3] 2024-09-16T09:40:34.278 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: metadata_pool 2 2024-09-16T09:40:34.278 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: inline_data disabled 2024-09-16T09:40:34.278 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: balancer 2024-09-16T09:40:34.278 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: bal_rank_mask -1 2024-09-16T09:40:34.278 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: standby_count_wanted 1 2024-09-16T09:40:34.278 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: qdb_cluster leader: 0 members: 2024-09-16T09:40:34.278 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: [mds.foofs.smithi078.hiotnn{0:14538} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.78:6834/245032473,v1:172.21.15.78:6835/245032473] compat {c=[1],r=[1],i=[7ff]}] 2024-09-16T09:40:34.278 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-16T09:40:34.278 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-16T09:40:34.278 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Standby daemons: 2024-09-16T09:40:34.278 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-16T09:40:34.278 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: [mds.foofs.smithi088.dbunob{-1:24279} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.88:6832/861752956,v1:172.21.15.88:6833/861752956] compat {c=[1],r=[1],i=[7ff]}] 2024-09-16T09:40:34.278 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: mon.smithi078@-1(???).osd e71 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-16T09:40:34.278 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: mon.smithi078@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-09-16T09:40:34.279 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: mon.smithi078@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-09-16T09:40:34.279 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: mon.smithi078@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-09-16T09:40:34.279 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: mon.smithi078@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-09-16T09:40:34.501 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 bash[63891]: f5a3a0fec7570aa41f61afb64ca6a348ab03e2bc531672b34a47a0135b944c31 2024-09-16T09:40:34.501 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 systemd[1]: Started Ceph mon.smithi078 for a70c765e-740d-11ef-bceb-c7b262605968. 2024-09-16T09:40:34.541 INFO:teuthology.orchestra.run.smithi078.stdout: 5 124076 10.81 MB/sec execute 174 sec latency 100.339 ms 2024-09-16T09:40:35.043 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: from='client.14910 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:40:35.043 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-16T09:40:35.043 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi088"}]: dispatch 2024-09-16T09:40:35.043 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: pgmap v27: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.8 MiB/s wr, 667 op/s 2024-09-16T09:40:35.043 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T09:40:35.043 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T09:40:35.043 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: from='client.14914 172.21.15.78:0/3087554644' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T09:40:35.044 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: from='client.24609 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:40:35.044 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/1290271044' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T09:40:35.044 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: pgmap v28: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.6 MiB/s wr, 456 op/s 2024-09-16T09:40:35.044 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: mon.smithi078 calling monitor election 2024-09-16T09:40:35.044 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: mon.smithi078 is new leader, mons smithi078,smithi088 in quorum (ranks 0,1) 2024-09-16T09:40:35.044 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: monmap epoch 2 2024-09-16T09:40:35.044 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: fsid a70c765e-740d-11ef-bceb-c7b262605968 2024-09-16T09:40:35.044 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: last_changed 2024-09-16T09:29:48.639329+0000 2024-09-16T09:40:35.044 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: created 2024-09-16T09:26:06.341058+0000 2024-09-16T09:40:35.044 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: min_mon_release 17 (quincy) 2024-09-16T09:40:35.044 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: election_strategy: 1 2024-09-16T09:40:35.044 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: 0: [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon.smithi078 2024-09-16T09:40:35.044 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: 1: [v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0] mon.smithi088 2024-09-16T09:40:35.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: fsmap foofs:1 {0=foofs.smithi078.hiotnn=up:active} 1 up:standby 2024-09-16T09:40:35.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: osdmap e71: 8 total, 8 up, 8 in 2024-09-16T09:40:35.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: mgrmap e35: smithi078.rrfxge(active, since 51s), standbys: smithi088.ncxfdz 2024-09-16T09:40:35.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: overall HEALTH_OK 2024-09-16T09:40:35.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: from='mgr.14844 ' entity='' 2024-09-16T09:40:35.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:34 smithi078 ceph-mon[63909]: mgrmap e36: smithi078.rrfxge(active, since 51s), standbys: smithi088.ncxfdz 2024-09-16T09:40:35.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:34 smithi088 ceph-mon[29151]: from='client.14910 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:40:35.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:34 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-16T09:40:35.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:34 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi088"}]: dispatch 2024-09-16T09:40:35.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:34 smithi088 ceph-mon[29151]: pgmap v27: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.8 MiB/s wr, 667 op/s 2024-09-16T09:40:35.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:34 smithi088 ceph-mon[29151]: from='mgr.14844 172.21.15.78:0/1806760722' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T09:40:35.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:34 smithi088 ceph-mon[29151]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T09:40:35.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:34 smithi088 ceph-mon[29151]: from='client.14914 172.21.15.78:0/3087554644' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T09:40:35.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:34 smithi088 ceph-mon[29151]: from='client.24609 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:40:35.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:34 smithi088 ceph-mon[29151]: from='client.? 172.21.15.78:0/1290271044' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T09:40:35.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:34 smithi088 ceph-mon[29151]: pgmap v28: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.6 MiB/s wr, 456 op/s 2024-09-16T09:40:35.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:34 smithi088 ceph-mon[29151]: mon.smithi078 calling monitor election 2024-09-16T09:40:35.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:34 smithi088 ceph-mon[29151]: mon.smithi078 is new leader, mons smithi078,smithi088 in quorum (ranks 0,1) 2024-09-16T09:40:35.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:34 smithi088 ceph-mon[29151]: monmap epoch 2 2024-09-16T09:40:35.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:34 smithi088 ceph-mon[29151]: fsid a70c765e-740d-11ef-bceb-c7b262605968 2024-09-16T09:40:35.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:34 smithi088 ceph-mon[29151]: last_changed 2024-09-16T09:29:48.639329+0000 2024-09-16T09:40:35.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:34 smithi088 ceph-mon[29151]: created 2024-09-16T09:26:06.341058+0000 2024-09-16T09:40:35.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:34 smithi088 ceph-mon[29151]: min_mon_release 17 (quincy) 2024-09-16T09:40:35.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:34 smithi088 ceph-mon[29151]: election_strategy: 1 2024-09-16T09:40:35.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:34 smithi088 ceph-mon[29151]: 0: [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon.smithi078 2024-09-16T09:40:35.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:34 smithi088 ceph-mon[29151]: 1: [v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0] mon.smithi088 2024-09-16T09:40:35.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:34 smithi088 ceph-mon[29151]: fsmap foofs:1 {0=foofs.smithi078.hiotnn=up:active} 1 up:standby 2024-09-16T09:40:35.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:34 smithi088 ceph-mon[29151]: osdmap e71: 8 total, 8 up, 8 in 2024-09-16T09:40:35.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:34 smithi088 ceph-mon[29151]: mgrmap e35: smithi078.rrfxge(active, since 51s), standbys: smithi088.ncxfdz 2024-09-16T09:40:35.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:34 smithi088 ceph-mon[29151]: overall HEALTH_OK 2024-09-16T09:40:35.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:34 smithi088 ceph-mon[29151]: from='mgr.14844 ' entity='' 2024-09-16T09:40:35.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:34 smithi088 ceph-mon[29151]: mgrmap e36: smithi078.rrfxge(active, since 51s), standbys: smithi088.ncxfdz 2024-09-16T09:40:35.542 INFO:teuthology.orchestra.run.smithi078.stdout: 5 124412 10.80 MB/sec execute 175 sec latency 41.107 ms 2024-09-16T09:40:36.542 INFO:teuthology.orchestra.run.smithi078.stdout: 5 124719 10.81 MB/sec execute 176 sec latency 53.998 ms 2024-09-16T09:40:36.878 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:36 smithi078 ceph-mon[63909]: mgrmap e37: smithi078.rrfxge(active, since 52s), standbys: smithi088.ncxfdz 2024-09-16T09:40:37.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:36 smithi088 ceph-mon[29151]: mgrmap e37: smithi078.rrfxge(active, since 52s), standbys: smithi088.ncxfdz 2024-09-16T09:40:37.542 INFO:teuthology.orchestra.run.smithi078.stdout: 5 124969 10.77 MB/sec execute 177 sec latency 52.819 ms 2024-09-16T09:40:38.542 INFO:teuthology.orchestra.run.smithi078.stdout: 5 125169 10.73 MB/sec execute 178 sec latency 51.868 ms 2024-09-16T09:40:39.542 INFO:teuthology.orchestra.run.smithi078.stdout: 5 125468 10.71 MB/sec execute 179 sec latency 76.836 ms 2024-09-16T09:40:40.542 INFO:teuthology.orchestra.run.smithi078.stdout: 5 125855 10.71 MB/sec execute 180 sec latency 43.130 ms 2024-09-16T09:40:41.200 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:40 smithi078 ceph-mon[63909]: Standby manager daemon smithi088.ncxfdz restarted 2024-09-16T09:40:41.201 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:40 smithi078 ceph-mon[63909]: Standby manager daemon smithi088.ncxfdz started 2024-09-16T09:40:41.201 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:40 smithi078 ceph-mon[63909]: from='mgr.? 172.21.15.88:0/788598622' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi088.ncxfdz/crt"}]: dispatch 2024-09-16T09:40:41.201 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:40 smithi078 ceph-mon[63909]: from='mgr.? 172.21.15.88:0/788598622' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-16T09:40:41.201 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:40 smithi078 ceph-mon[63909]: from='mgr.? 172.21.15.88:0/788598622' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi088.ncxfdz/key"}]: dispatch 2024-09-16T09:40:41.201 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:40 smithi078 ceph-mon[63909]: from='mgr.? 172.21.15.88:0/788598622' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-16T09:40:41.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:40 smithi088 ceph-mon[29151]: Standby manager daemon smithi088.ncxfdz restarted 2024-09-16T09:40:41.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:40 smithi088 ceph-mon[29151]: Standby manager daemon smithi088.ncxfdz started 2024-09-16T09:40:41.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:40 smithi088 ceph-mon[29151]: from='mgr.? 172.21.15.88:0/788598622' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi088.ncxfdz/crt"}]: dispatch 2024-09-16T09:40:41.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:40 smithi088 ceph-mon[29151]: from='mgr.? 172.21.15.88:0/788598622' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-16T09:40:41.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:40 smithi088 ceph-mon[29151]: from='mgr.? 172.21.15.88:0/788598622' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi088.ncxfdz/key"}]: dispatch 2024-09-16T09:40:41.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:40 smithi088 ceph-mon[29151]: from='mgr.? 172.21.15.88:0/788598622' entity='mgr.smithi088.ncxfdz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-16T09:40:41.542 INFO:teuthology.orchestra.run.smithi078.stdout: 5 126309 10.73 MB/sec execute 181 sec latency 32.651 ms 2024-09-16T09:40:42.218 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:41 smithi078 ceph-mon[63909]: mgrmap e38: smithi078.rrfxge(active, since 58s), standbys: smithi088.ncxfdz 2024-09-16T09:40:42.218 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:41 smithi078 ceph-mon[63909]: Active manager daemon smithi078.rrfxge restarted 2024-09-16T09:40:42.218 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:41 smithi078 ceph-mon[63909]: Activating manager daemon smithi078.rrfxge 2024-09-16T09:40:42.218 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:41 smithi078 ceph-mon[63909]: osdmap e72: 8 total, 8 up, 8 in 2024-09-16T09:40:42.219 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:41 smithi078 ceph-mon[63909]: mgrmap e39: smithi078.rrfxge(active, starting, since 0.00647467s), standbys: smithi088.ncxfdz 2024-09-16T09:40:42.219 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-16T09:40:42.219 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi088"}]: dispatch 2024-09-16T09:40:42.219 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.hiotnn"}]: dispatch 2024-09-16T09:40:42.219 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi088.dbunob"}]: dispatch 2024-09-16T09:40:42.219 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mgr metadata", "who": "smithi078.rrfxge", "id": "smithi078.rrfxge"}]: dispatch 2024-09-16T09:40:42.220 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mgr metadata", "who": "smithi088.ncxfdz", "id": "smithi088.ncxfdz"}]: dispatch 2024-09-16T09:40:42.220 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-16T09:40:42.220 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-16T09:40:42.220 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-16T09:40:42.220 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T09:40:42.220 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T09:40:42.220 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T09:40:42.220 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T09:40:42.220 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T09:40:42.221 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T09:40:42.221 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T09:40:42.221 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T09:40:42.221 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:41 smithi078 ceph-mon[63909]: Manager daemon smithi078.rrfxge is now available 2024-09-16T09:40:42.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:41 smithi088 ceph-mon[29151]: mgrmap e38: smithi078.rrfxge(active, since 58s), standbys: smithi088.ncxfdz 2024-09-16T09:40:42.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:41 smithi088 ceph-mon[29151]: Active manager daemon smithi078.rrfxge restarted 2024-09-16T09:40:42.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:41 smithi088 ceph-mon[29151]: Activating manager daemon smithi078.rrfxge 2024-09-16T09:40:42.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:41 smithi088 ceph-mon[29151]: osdmap e72: 8 total, 8 up, 8 in 2024-09-16T09:40:42.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:41 smithi088 ceph-mon[29151]: mgrmap e39: smithi078.rrfxge(active, starting, since 0.00647467s), standbys: smithi088.ncxfdz 2024-09-16T09:40:42.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:41 smithi088 ceph-mon[29151]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-16T09:40:42.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:41 smithi088 ceph-mon[29151]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata", "id": "smithi088"}]: dispatch 2024-09-16T09:40:42.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:41 smithi088 ceph-mon[29151]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.hiotnn"}]: dispatch 2024-09-16T09:40:42.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:41 smithi088 ceph-mon[29151]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi088.dbunob"}]: dispatch 2024-09-16T09:40:42.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:41 smithi088 ceph-mon[29151]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mgr metadata", "who": "smithi078.rrfxge", "id": "smithi078.rrfxge"}]: dispatch 2024-09-16T09:40:42.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:41 smithi088 ceph-mon[29151]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mgr metadata", "who": "smithi088.ncxfdz", "id": "smithi088.ncxfdz"}]: dispatch 2024-09-16T09:40:42.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:41 smithi088 ceph-mon[29151]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-16T09:40:42.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:41 smithi088 ceph-mon[29151]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-16T09:40:42.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:41 smithi088 ceph-mon[29151]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-16T09:40:42.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:41 smithi088 ceph-mon[29151]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T09:40:42.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:41 smithi088 ceph-mon[29151]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T09:40:42.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:41 smithi088 ceph-mon[29151]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T09:40:42.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:41 smithi088 ceph-mon[29151]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T09:40:42.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:41 smithi088 ceph-mon[29151]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T09:40:42.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:41 smithi088 ceph-mon[29151]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T09:40:42.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:41 smithi088 ceph-mon[29151]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T09:40:42.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:41 smithi088 ceph-mon[29151]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T09:40:42.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:41 smithi088 ceph-mon[29151]: Manager daemon smithi078.rrfxge is now available 2024-09-16T09:40:42.542 INFO:teuthology.orchestra.run.smithi078.stdout: 5 126654 10.68 MB/sec execute 182 sec latency 55.634 ms 2024-09-16T09:40:43.226 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:42 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:40:43.227 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:42 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T09:40:43.227 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:42 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T09:40:43.227 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:42 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:40:43.227 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:42 smithi078 ceph-mon[63909]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/mirror_snapshot_schedule"} : dispatch 2024-09-16T09:40:43.227 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:42 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/trash_purge_schedule"}]: dispatch 2024-09-16T09:40:43.227 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:42 smithi078 ceph-mon[63909]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/trash_purge_schedule"} : dispatch 2024-09-16T09:40:43.227 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:42 smithi078 ceph-mon[63909]: mgrmap e40: smithi078.rrfxge(active, since 1.03333s), standbys: smithi088.ncxfdz 2024-09-16T09:40:43.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:42 smithi088 ceph-mon[29151]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:40:43.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:42 smithi088 ceph-mon[29151]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-16T09:40:43.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:42 smithi088 ceph-mon[29151]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T09:40:43.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:42 smithi088 ceph-mon[29151]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/mirror_snapshot_schedule"}]: dispatch 2024-09-16T09:40:43.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:42 smithi088 ceph-mon[29151]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/mirror_snapshot_schedule"} : dispatch 2024-09-16T09:40:43.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:42 smithi088 ceph-mon[29151]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/trash_purge_schedule"}]: dispatch 2024-09-16T09:40:43.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:42 smithi088 ceph-mon[29151]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.rrfxge/trash_purge_schedule"} : dispatch 2024-09-16T09:40:43.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:42 smithi088 ceph-mon[29151]: mgrmap e40: smithi078.rrfxge(active, since 1.03333s), standbys: smithi088.ncxfdz 2024-09-16T09:40:43.542 INFO:teuthology.orchestra.run.smithi078.stdout: 5 127023 10.67 MB/sec execute 183 sec latency 43.982 ms 2024-09-16T09:40:44.249 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:43 smithi078 ceph-mon[63909]: [16/Sep/2024:09:40:43] ENGINE Bus STARTING 2024-09-16T09:40:44.249 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:43 smithi078 ceph-mon[63909]: [16/Sep/2024:09:40:43] ENGINE Serving on https://172.21.15.78:7150 2024-09-16T09:40:44.249 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:43 smithi078 ceph-mon[63909]: [16/Sep/2024:09:40:43] ENGINE Error in HTTPServer.serve 2024-09-16T09:40:44.249 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Traceback (most recent call last): 2024-09-16T09:40:44.249 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-16T09:40:44.249 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self._connections.run(self.expiration_interval) 2024-09-16T09:40:44.249 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-16T09:40:44.249 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self._run(expiration_interval) 2024-09-16T09:40:44.249 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-16T09:40:44.249 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-16T09:40:44.249 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-16T09:40:44.249 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-16T09:40:44.250 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-16T09:40:44.250 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: s = self.context.wrap_socket( 2024-09-16T09:40:44.250 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-16T09:40:44.250 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: return self.sslsocket_class._create( 2024-09-16T09:40:44.250 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-16T09:40:44.250 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self.do_handshake() 2024-09-16T09:40:44.250 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-16T09:40:44.250 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self._sslobj.do_handshake() 2024-09-16T09:40:44.250 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-16T09:40:44.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:43 smithi088 ceph-mon[29151]: [16/Sep/2024:09:40:43] ENGINE Bus STARTING 2024-09-16T09:40:44.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:43 smithi088 ceph-mon[29151]: [16/Sep/2024:09:40:43] ENGINE Serving on https://172.21.15.78:7150 2024-09-16T09:40:44.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:43 smithi088 ceph-mon[29151]: [16/Sep/2024:09:40:43] ENGINE Error in HTTPServer.serve 2024-09-16T09:40:44.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Traceback (most recent call last): 2024-09-16T09:40:44.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-16T09:40:44.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: self._connections.run(self.expiration_interval) 2024-09-16T09:40:44.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-16T09:40:44.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: self._run(expiration_interval) 2024-09-16T09:40:44.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-16T09:40:44.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-16T09:40:44.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-16T09:40:44.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-16T09:40:44.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-16T09:40:44.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: s = self.context.wrap_socket( 2024-09-16T09:40:44.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-16T09:40:44.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: return self.sslsocket_class._create( 2024-09-16T09:40:44.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-16T09:40:44.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: self.do_handshake() 2024-09-16T09:40:44.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-16T09:40:44.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: self._sslobj.do_handshake() 2024-09-16T09:40:44.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-16T09:40:44.542 INFO:teuthology.orchestra.run.smithi078.stdout: 5 127211 10.63 MB/sec execute 184 sec latency 48.172 ms 2024-09-16T09:40:45.031 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:44 smithi078 ceph-mon[63909]: [16/Sep/2024:09:40:43] ENGINE Serving on http://172.21.15.78:8765 2024-09-16T09:40:45.032 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:44 smithi078 ceph-mon[63909]: [16/Sep/2024:09:40:43] ENGINE Bus STARTED 2024-09-16T09:40:45.032 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:44 smithi078 ceph-mon[63909]: pgmap v4: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-16T09:40:45.032 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:44 smithi078 ceph-mon[63909]: mgrmap e41: smithi078.rrfxge(active, since 2s), standbys: smithi088.ncxfdz 2024-09-16T09:40:45.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:44 smithi088 ceph-mon[29151]: [16/Sep/2024:09:40:43] ENGINE Serving on http://172.21.15.78:8765 2024-09-16T09:40:45.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:44 smithi088 ceph-mon[29151]: [16/Sep/2024:09:40:43] ENGINE Bus STARTED 2024-09-16T09:40:45.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:44 smithi088 ceph-mon[29151]: pgmap v4: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-16T09:40:45.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:44 smithi088 ceph-mon[29151]: mgrmap e41: smithi078.rrfxge(active, since 2s), standbys: smithi088.ncxfdz 2024-09-16T09:40:45.542 INFO:teuthology.orchestra.run.smithi078.stdout: 5 127593 10.68 MB/sec execute 185 sec latency 58.856 ms 2024-09-16T09:40:46.542 INFO:teuthology.orchestra.run.smithi078.stdout: 5 127904 10.67 MB/sec execute 186 sec latency 51.771 ms 2024-09-16T09:40:47.112 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:47 smithi078 ceph-mon[63909]: pgmap v5: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-16T09:40:47.113 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:47 smithi078 ceph-mon[63909]: mgrmap e42: smithi078.rrfxge(active, since 4s), standbys: smithi088.ncxfdz 2024-09-16T09:40:47.113 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:47 smithi078 ceph-mon[63909]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:47.113 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:47 smithi078 ceph-mon[63909]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:47.542 INFO:teuthology.orchestra.run.smithi078.stdout: 5 128216 10.68 MB/sec execute 187 sec latency 92.061 ms 2024-09-16T09:40:47.550 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:47 smithi088 ceph-mon[29151]: pgmap v5: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-16T09:40:47.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:47 smithi088 ceph-mon[29151]: mgrmap e42: smithi078.rrfxge(active, since 4s), standbys: smithi088.ncxfdz 2024-09-16T09:40:47.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:47 smithi088 ceph-mon[29151]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:47.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:47 smithi088 ceph-mon[29151]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:48.542 INFO:teuthology.orchestra.run.smithi078.stdout: 5 128460 10.65 MB/sec execute 188 sec latency 51.273 ms 2024-09-16T09:40:48.635 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:48 smithi078 ceph-mon[63909]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:48.635 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:48 smithi078 ceph-mon[63909]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:48.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:48 smithi088 ceph-mon[29151]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:48.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:48 smithi088 ceph-mon[29151]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:49.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:49 smithi078 ceph-mon[63909]: pgmap v6: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-16T09:40:49.448 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:49 smithi088 ceph-mon[29151]: pgmap v6: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-16T09:40:49.542 INFO:teuthology.orchestra.run.smithi078.stdout: 5 128631 10.61 MB/sec execute 189 sec latency 110.820 ms 2024-09-16T09:40:50.542 INFO:teuthology.orchestra.run.smithi078.stdout: 5 128889 10.58 MB/sec execute 190 sec latency 53.890 ms 2024-09-16T09:40:51.542 INFO:teuthology.orchestra.run.smithi078.stdout: 5 129160 10.56 MB/sec execute 191 sec latency 60.850 ms 2024-09-16T09:40:51.956 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:51 smithi078 ceph-mon[63909]: pgmap v7: 97 pgs: 97 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 30 KiB/s rd, 3.7 MiB/s wr, 321 op/s 2024-09-16T09:40:51.956 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:51 smithi078 ceph-mon[63909]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:51.956 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:51 smithi078 ceph-mon[63909]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:51.956 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:51 smithi078 ceph-mon[63909]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:51.956 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:51 smithi078 ceph-mon[63909]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:51.956 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:51 smithi078 ceph-mon[63909]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:51.956 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:51 smithi078 ceph-mon[63909]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:51.957 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:51 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:40:51.957 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:51 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:40:52.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:51 smithi088 ceph-mon[29151]: pgmap v7: 97 pgs: 97 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 30 KiB/s rd, 3.7 MiB/s wr, 321 op/s 2024-09-16T09:40:52.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:51 smithi088 ceph-mon[29151]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:52.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:51 smithi088 ceph-mon[29151]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:52.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:51 smithi088 ceph-mon[29151]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:52.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:51 smithi088 ceph-mon[29151]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:52.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:51 smithi088 ceph-mon[29151]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:52.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:51 smithi088 ceph-mon[29151]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:52.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:51 smithi088 ceph-mon[29151]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:40:52.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:51 smithi088 ceph-mon[29151]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T09:40:52.542 INFO:teuthology.orchestra.run.smithi078.stdout: 5 129366 10.53 MB/sec execute 192 sec latency 70.345 ms 2024-09-16T09:40:52.976 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:52 smithi078 ceph-mon[63909]: Updating smithi078:/etc/ceph/ceph.conf 2024-09-16T09:40:52.976 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:52 smithi078 ceph-mon[63909]: Updating smithi088:/etc/ceph/ceph.conf 2024-09-16T09:40:52.976 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:52 smithi078 ceph-mon[63909]: Updating smithi088:/var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-16T09:40:52.976 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:52 smithi078 ceph-mon[63909]: Updating smithi078:/var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-16T09:40:52.976 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:52 smithi078 ceph-mon[63909]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:52.976 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:52 smithi078 ceph-mon[63909]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:52.976 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:52 smithi078 ceph-mon[63909]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:52.976 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:52 smithi078 ceph-mon[63909]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:52.976 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:52 smithi078 ceph-mon[63909]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:52.977 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:52 smithi078 ceph-mon[63909]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:52.977 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:52 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:40:52.977 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:52 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T09:40:52.977 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:52 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-16T09:40:53.059 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:52 smithi088 ceph-mon[29151]: Updating smithi078:/etc/ceph/ceph.conf 2024-09-16T09:40:53.059 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:52 smithi088 ceph-mon[29151]: Updating smithi088:/etc/ceph/ceph.conf 2024-09-16T09:40:53.059 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:52 smithi088 ceph-mon[29151]: Updating smithi088:/var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-16T09:40:53.059 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:52 smithi088 ceph-mon[29151]: Updating smithi078:/var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-16T09:40:53.060 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:52 smithi088 ceph-mon[29151]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:53.060 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:52 smithi088 ceph-mon[29151]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:53.060 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:52 smithi088 ceph-mon[29151]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:53.060 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:52 smithi088 ceph-mon[29151]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:53.060 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:52 smithi088 ceph-mon[29151]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:53.060 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:52 smithi088 ceph-mon[29151]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:53.060 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:52 smithi088 ceph-mon[29151]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T09:40:53.060 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:52 smithi088 ceph-mon[29151]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T09:40:53.060 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:52 smithi088 ceph-mon[29151]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-16T09:40:53.542 INFO:teuthology.orchestra.run.smithi078.stdout: 5 129687 10.52 MB/sec execute 193 sec latency 58.043 ms 2024-09-16T09:40:54.003 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:53 smithi078 ceph-mon[63909]: pgmap v8: 97 pgs: 97 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 2.8 MiB/s wr, 249 op/s 2024-09-16T09:40:54.004 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:53 smithi078 ceph-mon[63909]: Updating smithi088:/etc/ceph/ceph.client.admin.keyring 2024-09-16T09:40:54.004 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:53 smithi078 ceph-mon[63909]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-09-16T09:40:54.004 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:53 smithi078 ceph-mon[63909]: Updating smithi088:/var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-16T09:40:54.004 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:53 smithi078 ceph-mon[63909]: Updating smithi078:/var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-16T09:40:54.278 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:53 smithi088 ceph-mon[29151]: pgmap v8: 97 pgs: 97 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 2.8 MiB/s wr, 249 op/s 2024-09-16T09:40:54.278 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:53 smithi088 ceph-mon[29151]: Updating smithi088:/etc/ceph/ceph.client.admin.keyring 2024-09-16T09:40:54.279 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:53 smithi088 ceph-mon[29151]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-09-16T09:40:54.279 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:53 smithi088 ceph-mon[29151]: Updating smithi088:/var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-16T09:40:54.279 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:53 smithi088 ceph-mon[29151]: Updating smithi078:/var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-16T09:40:54.542 INFO:teuthology.orchestra.run.smithi078.stdout: 5 129950 10.50 MB/sec execute 194 sec latency 70.716 ms 2024-09-16T09:40:55.543 INFO:teuthology.orchestra.run.smithi078.stdout: 5 130326 10.46 MB/sec execute 195 sec latency 55.931 ms 2024-09-16T09:40:55.575 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:55 smithi078 ceph-mon[63909]: pgmap v9: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 4.2 MiB/s wr, 388 op/s 2024-09-16T09:40:55.575 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:55 smithi078 ceph-mon[63909]: Upgrade: Updating mon.smithi088 2024-09-16T09:40:55.575 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:55 smithi078 ceph-mon[63909]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:55.575 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:55 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-16T09:40:55.575 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:55 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-16T09:40:55.575 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:55 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:40:55.575 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:55 smithi078 ceph-mon[63909]: Deploying daemon mon.smithi088 on smithi088 2024-09-16T09:40:55.770 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:55 smithi088 ceph-mon[29151]: pgmap v9: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 4.2 MiB/s wr, 388 op/s 2024-09-16T09:40:55.770 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:55 smithi088 ceph-mon[29151]: Upgrade: Updating mon.smithi088 2024-09-16T09:40:55.770 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:55 smithi088 ceph-mon[29151]: from='mgr.24621 ' entity='mgr.smithi078.rrfxge' 2024-09-16T09:40:55.770 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:55 smithi088 ceph-mon[29151]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-16T09:40:55.770 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:55 smithi088 ceph-mon[29151]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-16T09:40:55.771 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:55 smithi088 ceph-mon[29151]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T09:40:55.771 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:55 smithi088 ceph-mon[29151]: Deploying daemon mon.smithi088 on smithi088 2024-09-16T09:40:56.448 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:56 smithi088 systemd[1]: Stopping Ceph mon.smithi088 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:40:56.542 INFO:teuthology.orchestra.run.smithi078.stdout: 5 130603 10.45 MB/sec execute 196 sec latency 137.056 ms 2024-09-16T09:40:56.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:56 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi088[29132]: 2024-09-16T09:40:56.442+0000 7fe77bc6d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi088 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T09:40:56.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:56 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi088[29132]: 2024-09-16T09:40:56.442+0000 7fe77bc6d700 -1 mon.smithi088@1(peon) e2 *** Got Signal Terminated *** 2024-09-16T09:40:56.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:56 smithi088 podman[52609]: 2024-09-16 09:40:56.464322373 +0000 UTC m=+0.164708987 container died 774a14c001137d19729dd3c5b34889d2f9899d5b6fd9349c7cfefee18320bd7c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi088, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., RELEASE=HEAD, ceph=True, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, name=centos-stream, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, version=8, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-16T09:40:57.530 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:57 smithi088 podman[52609]: 2024-09-16 09:40:57.285834812 +0000 UTC m=+0.986221424 container cleanup 774a14c001137d19729dd3c5b34889d2f9899d5b6fd9349c7cfefee18320bd7c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi088, GIT_CLEAN=True, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, vendor=Red Hat, Inc., io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , release=754, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, vcs-type=git, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, GIT_BRANCH=HEAD) 2024-09-16T09:40:57.531 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:57 smithi088 bash[52609]: ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi088 2024-09-16T09:40:57.542 INFO:teuthology.orchestra.run.smithi078.stdout: 5 130799 10.43 MB/sec execute 197 sec latency 74.144 ms 2024-09-16T09:40:57.794 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:57 smithi088 podman[52621]: 2024-09-16 09:40:57.529519279 +0000 UTC m=+1.060019744 container remove 774a14c001137d19729dd3c5b34889d2f9899d5b6fd9349c7cfefee18320bd7c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi088, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, RELEASE=HEAD, io.openshift.expose-services=, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , vcs-type=git, com.redhat.component=centos-stream-container, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-16T09:40:58.057 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:57 smithi088 systemd[1]: ceph-a70c765e-740d-11ef-bceb-c7b262605968@mon.smithi088.service: Deactivated successfully. 2024-09-16T09:40:58.058 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:57 smithi088 systemd[1]: Stopped Ceph mon.smithi088 for a70c765e-740d-11ef-bceb-c7b262605968. 2024-09-16T09:40:58.058 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:57 smithi088 systemd[1]: ceph-a70c765e-740d-11ef-bceb-c7b262605968@mon.smithi088.service: Consumed 7.120s CPU time. 2024-09-16T09:40:58.398 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 systemd[1]: Starting Ceph mon.smithi088 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:40:58.398 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 podman[52740]: 2024-09-16 09:40:58.197447578 +0000 UTC m=+0.020829491 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:40:58.542 INFO:teuthology.orchestra.run.smithi078.stdout: 5 131148 10.46 MB/sec execute 198 sec latency 50.062 ms 2024-09-16T09:40:58.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 podman[52740]: 2024-09-16 09:40:58.397308936 +0000 UTC m=+0.220690838 container create 2e552147327ac311d346dc2bc8c6607195094064e66158a49b14e9748bd9b89c (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi088, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD) 2024-09-16T09:40:59.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 podman[52740]: 2024-09-16 09:40:58.867724385 +0000 UTC m=+0.691106300 container init 2e552147327ac311d346dc2bc8c6607195094064e66158a49b14e9748bd9b89c (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi088, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-16T09:40:59.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 podman[52740]: 2024-09-16 09:40:58.873045808 +0000 UTC m=+0.696427714 container start 2e552147327ac311d346dc2bc8c6607195094064e66158a49b14e9748bd9b89c (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi088, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-16T09:40:59.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: set uid:gid to 167:167 (ceph:ceph) 2024-09-16T09:40:59.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev), process ceph-mon, pid 2 2024-09-16T09:40:59.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: pidfile_write: ignore empty --pid-file 2024-09-16T09:40:59.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: load: jerasure load: lrc 2024-09-16T09:40:59.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: RocksDB version: 7.9.2 2024-09-16T09:40:59.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Git sha 0 2024-09-16T09:40:59.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Compile date 2024-09-12 18:19:39 2024-09-16T09:40:59.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: DB SUMMARY 2024-09-16T09:40:59.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: DB Session ID: DF3B6F7YGEF46AGPA0GD 2024-09-16T09:40:59.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: CURRENT file: CURRENT 2024-09-16T09:40:59.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: IDENTITY file: IDENTITY 2024-09-16T09:40:59.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: MANIFEST file: MANIFEST-000009 size: 882 Bytes 2024-09-16T09:40:59.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi088/store.db dir, Total Num: 1, files: 000024.sst 2024-09-16T09:40:59.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi088/store.db: 000022.log size: 9512197 ; 2024-09-16T09:40:59.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.error_if_exists: 0 2024-09-16T09:40:59.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.create_if_missing: 0 2024-09-16T09:40:59.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.paranoid_checks: 1 2024-09-16T09:40:59.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-16T09:40:59.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-16T09:40:59.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-16T09:40:59.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.env: 0x55599c24d120 2024-09-16T09:40:59.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.fs: PosixFileSystem 2024-09-16T09:40:59.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.info_log: 0x55599e3df8e0 2024-09-16T09:40:59.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.max_file_opening_threads: 16 2024-09-16T09:40:59.304 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.statistics: (nil) 2024-09-16T09:40:59.305 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.use_fsync: 0 2024-09-16T09:40:59.305 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.max_log_file_size: 0 2024-09-16T09:40:59.305 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-16T09:40:59.305 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-16T09:40:59.305 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.keep_log_file_num: 1000 2024-09-16T09:40:59.305 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.recycle_log_file_num: 0 2024-09-16T09:40:59.305 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.allow_fallocate: 1 2024-09-16T09:40:59.305 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.allow_mmap_reads: 0 2024-09-16T09:40:59.305 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.allow_mmap_writes: 0 2024-09-16T09:40:59.305 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.use_direct_reads: 0 2024-09-16T09:40:59.305 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-16T09:40:59.306 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.create_missing_column_families: 0 2024-09-16T09:40:59.306 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.db_log_dir: 2024-09-16T09:40:59.306 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.wal_dir: 2024-09-16T09:40:59.306 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-16T09:40:59.306 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-16T09:40:59.306 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-16T09:40:59.306 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-16T09:40:59.306 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-16T09:40:59.306 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-16T09:40:59.306 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.advise_random_on_open: 1 2024-09-16T09:40:59.306 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.db_write_buffer_size: 0 2024-09-16T09:40:59.306 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.write_buffer_manager: 0x55599e3e3b80 2024-09-16T09:40:59.306 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-16T09:40:59.307 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-16T09:40:59.307 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-16T09:40:59.307 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.rate_limiter: (nil) 2024-09-16T09:40:59.307 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-16T09:40:59.307 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.wal_recovery_mode: 2 2024-09-16T09:40:59.307 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.enable_thread_tracking: 0 2024-09-16T09:40:59.307 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.enable_pipelined_write: 0 2024-09-16T09:40:59.307 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.unordered_write: 0 2024-09-16T09:40:59.307 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-16T09:40:59.307 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-16T09:40:59.307 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-16T09:40:59.307 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-16T09:40:59.307 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.row_cache: None 2024-09-16T09:40:59.307 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.wal_filter: None 2024-09-16T09:40:59.308 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-16T09:40:59.308 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.allow_ingest_behind: 0 2024-09-16T09:40:59.308 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.two_write_queues: 0 2024-09-16T09:40:59.308 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.manual_wal_flush: 0 2024-09-16T09:40:59.308 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.wal_compression: 0 2024-09-16T09:40:59.308 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.atomic_flush: 0 2024-09-16T09:40:59.308 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-16T09:40:59.308 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-16T09:40:59.308 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-16T09:40:59.308 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.log_readahead_size: 0 2024-09-16T09:40:59.308 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-16T09:40:59.308 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.best_efforts_recovery: 0 2024-09-16T09:40:59.308 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-16T09:40:59.309 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-16T09:40:59.309 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.allow_data_in_errors: 0 2024-09-16T09:40:59.309 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.db_host_id: __hostname__ 2024-09-16T09:40:59.309 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-16T09:40:59.309 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.max_background_jobs: 2 2024-09-16T09:40:59.309 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.max_background_compactions: -1 2024-09-16T09:40:59.309 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.max_subcompactions: 1 2024-09-16T09:40:59.310 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-16T09:40:59.310 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-16T09:40:59.310 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-16T09:40:59.310 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.max_total_wal_size: 0 2024-09-16T09:40:59.310 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-16T09:40:59.310 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-16T09:40:59.310 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-16T09:40:59.311 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-16T09:40:59.311 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.max_open_files: -1 2024-09-16T09:40:59.311 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.bytes_per_sync: 0 2024-09-16T09:40:59.311 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-16T09:40:59.311 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-16T09:40:59.311 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.compaction_readahead_size: 0 2024-09-16T09:40:59.311 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.max_background_flushes: -1 2024-09-16T09:40:59.311 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Compression algorithms supported: 2024-09-16T09:40:59.312 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: kZSTD supported: 0 2024-09-16T09:40:59.312 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: kXpressCompression supported: 0 2024-09-16T09:40:59.312 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: kBZip2Compression supported: 0 2024-09-16T09:40:59.312 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-16T09:40:59.312 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: kLZ4Compression supported: 1 2024-09-16T09:40:59.312 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: kZlibCompression supported: 1 2024-09-16T09:40:59.312 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: kLZ4HCCompression supported: 1 2024-09-16T09:40:59.312 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: kSnappyCompression supported: 1 2024-09-16T09:40:59.312 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-16T09:40:59.312 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-16T09:40:59.312 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi088/store.db/MANIFEST-000009 2024-09-16T09:40:59.312 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-16T09:40:59.312 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-16T09:40:59.313 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.merge_operator: 2024-09-16T09:40:59.313 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.compaction_filter: None 2024-09-16T09:40:59.313 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.compaction_filter_factory: None 2024-09-16T09:40:59.313 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.sst_partitioner_factory: None 2024-09-16T09:40:59.313 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-16T09:40:59.313 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-16T09:40:59.313 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55599e3df540) 2024-09-16T09:40:59.313 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: cache_index_and_filter_blocks: 1 2024-09-16T09:40:59.313 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-16T09:40:59.313 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-16T09:40:59.314 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: pin_top_level_index_and_filter: 1 2024-09-16T09:40:59.314 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: index_type: 0 2024-09-16T09:40:59.314 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: data_block_index_type: 0 2024-09-16T09:40:59.314 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: index_shortening: 1 2024-09-16T09:40:59.314 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-16T09:40:59.314 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: checksum: 4 2024-09-16T09:40:59.314 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: no_block_cache: 0 2024-09-16T09:40:59.314 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: block_cache: 0x55599e3d3610 2024-09-16T09:40:59.314 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: block_cache_name: BinnedLRUCache 2024-09-16T09:40:59.314 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: block_cache_options: 2024-09-16T09:40:59.314 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: capacity : 536870912 2024-09-16T09:40:59.314 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: num_shard_bits : 4 2024-09-16T09:40:59.314 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: strict_capacity_limit : 0 2024-09-16T09:40:59.314 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: high_pri_pool_ratio: 0.000 2024-09-16T09:40:59.315 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: block_cache_compressed: (nil) 2024-09-16T09:40:59.315 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: persistent_cache: (nil) 2024-09-16T09:40:59.315 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: block_size: 4096 2024-09-16T09:40:59.315 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: block_size_deviation: 10 2024-09-16T09:40:59.315 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: block_restart_interval: 16 2024-09-16T09:40:59.315 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: index_block_restart_interval: 1 2024-09-16T09:40:59.315 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: metadata_block_size: 4096 2024-09-16T09:40:59.315 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: partition_filters: 0 2024-09-16T09:40:59.315 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: use_delta_encoding: 1 2024-09-16T09:40:59.315 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: filter_policy: bloomfilter 2024-09-16T09:40:59.315 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: whole_key_filtering: 1 2024-09-16T09:40:59.315 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: verify_compression: 0 2024-09-16T09:40:59.316 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: read_amp_bytes_per_bit: 0 2024-09-16T09:40:59.316 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: format_version: 5 2024-09-16T09:40:59.316 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: enable_index_compression: 1 2024-09-16T09:40:59.316 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: block_align: 0 2024-09-16T09:40:59.316 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: max_auto_readahead_size: 262144 2024-09-16T09:40:59.316 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: prepopulate_block_cache: 0 2024-09-16T09:40:59.316 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: initial_auto_readahead_size: 8192 2024-09-16T09:40:59.316 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: num_file_reads_for_auto_readahead: 2 2024-09-16T09:40:59.316 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.write_buffer_size: 33554432 2024-09-16T09:40:59.316 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.max_write_buffer_number: 2 2024-09-16T09:40:59.316 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.compression: NoCompression 2024-09-16T09:40:59.316 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.bottommost_compression: Disabled 2024-09-16T09:40:59.317 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.prefix_extractor: nullptr 2024-09-16T09:40:59.317 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-16T09:40:59.317 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.num_levels: 7 2024-09-16T09:40:59.317 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-16T09:40:59.317 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-16T09:40:59.317 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-16T09:40:59.317 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-16T09:40:59.317 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-16T09:40:59.317 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-16T09:40:59.317 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-16T09:40:59.317 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-16T09:40:59.317 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-16T09:40:59.317 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-16T09:40:59.318 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-16T09:40:59.318 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-16T09:40:59.318 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-16T09:40:59.318 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.compression_opts.level: 32767 2024-09-16T09:40:59.318 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.compression_opts.strategy: 0 2024-09-16T09:40:59.318 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-16T09:40:59.318 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-16T09:40:59.318 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-16T09:40:59.318 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-16T09:40:59.318 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.compression_opts.enabled: false 2024-09-16T09:40:59.318 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-16T09:40:59.318 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-16T09:40:59.318 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-16T09:40:59.318 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-16T09:40:59.319 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.target_file_size_base: 67108864 2024-09-16T09:40:59.319 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-16T09:40:59.319 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-16T09:40:59.319 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-16T09:40:59.319 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-16T09:40:59.319 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-16T09:40:59.319 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-16T09:40:59.319 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-16T09:40:59.319 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-16T09:40:59.319 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-16T09:40:59.319 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-16T09:40:59.319 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-16T09:40:59.319 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-16T09:40:59.319 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-16T09:40:59.320 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-16T09:40:59.320 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.arena_block_size: 1048576 2024-09-16T09:40:59.320 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-16T09:40:59.320 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-16T09:40:59.320 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.disable_auto_compactions: 0 2024-09-16T09:40:59.320 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-16T09:40:59.320 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-16T09:40:59.320 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-16T09:40:59.320 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-16T09:40:59.320 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-16T09:40:59.320 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-16T09:40:59.321 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-16T09:40:59.321 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-16T09:40:59.321 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-16T09:40:59.321 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-16T09:40:59.321 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-16T09:40:59.321 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.inplace_update_support: 0 2024-09-16T09:40:59.322 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-16T09:40:59.322 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-16T09:40:59.322 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-16T09:40:59.322 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-16T09:40:59.322 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.bloom_locality: 0 2024-09-16T09:40:59.322 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.max_successive_merges: 0 2024-09-16T09:40:59.322 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-16T09:40:59.322 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.paranoid_file_checks: 0 2024-09-16T09:40:59.323 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.force_consistency_checks: 1 2024-09-16T09:40:59.323 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.report_bg_io_stats: 0 2024-09-16T09:40:59.323 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.ttl: 2592000 2024-09-16T09:40:59.323 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-16T09:40:59.323 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-16T09:40:59.324 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-16T09:40:59.324 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.enable_blob_files: false 2024-09-16T09:40:59.324 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.min_blob_size: 0 2024-09-16T09:40:59.324 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.blob_file_size: 268435456 2024-09-16T09:40:59.324 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-16T09:40:59.324 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-16T09:40:59.324 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-16T09:40:59.324 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-16T09:40:59.324 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-16T09:40:59.324 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.blob_file_starting_level: 0 2024-09-16T09:40:59.325 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-16T09:40:59.325 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 24.sst 2024-09-16T09:40:59.325 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-16T09:40:59.325 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi088/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 26, last_sequence is 11723, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-16T09:40:59.325 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2024-09-16T09:40:59.325 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 0e6afdfc-379d-48b8-a541-bc891acafc01 2024-09-16T09:40:59.325 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726479658903934, "job": 1, "event": "recovery_started", "wal_files": [22]} 2024-09-16T09:40:59.325 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2024-09-16T09:40:59.325 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726479658938272, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 5780723, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11712, "largest_seqno": 14901, "table_properties": {"data_size": 5768297, "index_size": 7987, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 3397, "raw_key_size": 29822, "raw_average_key_size": 22, "raw_value_size": 5741104, "raw_average_value_size": 4236, "num_data_blocks": 358, "num_entries": 1355, "num_filter_entries": 1355, "num_deletions": 253, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726479658, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "0e6afdfc-379d-48b8-a541-bc891acafc01", "db_session_id": "DF3B6F7YGEF46AGPA0GD", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2024-09-16T09:40:59.325 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726479658938410, "job": 1, "event": "recovery_finished"} 2024-09-16T09:40:59.325 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2024-09-16T09:40:59.326 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-16T09:40:59.326 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi088/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-16T09:40:59.326 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55599e402e00 2024-09-16T09:40:59.326 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: DB pointer 0x55599e504000 2024-09-16T09:40:59.326 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-16T09:40:59.326 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-16T09:40:59.327 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: ** DB Stats ** 2024-09-16T09:40:59.327 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-16T09:40:59.327 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-16T09:40:59.327 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-16T09:40:59.327 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-16T09:40:59.327 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-16T09:40:59.327 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-16T09:40:59.328 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-16T09:40:59.328 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: 2024-09-16T09:40:59.328 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: ** Compaction Stats [default] ** 2024-09-16T09:40:59.328 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-16T09:40:59.328 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-16T09:40:59.328 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: L0 1/0 5.51 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 374.2 0.01 0.00 1 0.015 0 0 0.0 0.0 2024-09-16T09:40:59.329 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: L6 1/0 6.40 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-16T09:40:59.329 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Sum 2/0 11.91 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 374.2 0.01 0.00 1 0.015 0 0 0.0 0.0 2024-09-16T09:40:59.329 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 374.2 0.01 0.00 1 0.015 0 0 0.0 0.0 2024-09-16T09:40:59.329 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: 2024-09-16T09:40:59.329 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: ** Compaction Stats [default] ** 2024-09-16T09:40:59.329 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-16T09:40:59.329 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-16T09:40:59.329 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 374.2 0.01 0.00 1 0.015 0 0 0.0 0.0 2024-09-16T09:40:59.329 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: 2024-09-16T09:40:59.330 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-16T09:40:59.330 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: 2024-09-16T09:40:59.330 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-16T09:40:59.330 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Flush(GB): cumulative 0.005, interval 0.005 2024-09-16T09:40:59.330 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-16T09:40:59.330 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-16T09:40:59.330 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-16T09:40:59.330 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-16T09:40:59.330 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Cumulative compaction: 0.01 GB write, 144.85 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-16T09:40:59.330 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Interval compaction: 0.01 GB write, 144.85 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-16T09:40:59.330 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-16T09:40:59.330 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Block cache BinnedLRUCache@0x55599e3d3610#2 capacity: 512.00 MB usage: 12.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.4e-05 secs_since: 0 2024-09-16T09:40:59.331 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,4.03 KB,0.0007689%) IndexBlock(1,8.08 KB,0.00154078%) Misc(1,0.00 KB,0%) 2024-09-16T09:40:59.331 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: 2024-09-16T09:40:59.331 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-16T09:40:59.331 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: starting mon.smithi088 rank 1 at public addrs [v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0] at bind addrs [v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi088 fsid a70c765e-740d-11ef-bceb-c7b262605968 2024-09-16T09:40:59.331 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: mon.smithi088@-1(???) e2 preinit fsid a70c765e-740d-11ef-bceb-c7b262605968 2024-09-16T09:40:59.331 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: mon.smithi088@-1(???).mds e7 new map 2024-09-16T09:40:59.332 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: mon.smithi088@-1(???).mds e7 print_map 2024-09-16T09:40:59.332 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: e7 2024-09-16T09:40:59.332 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-16T09:40:59.332 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-16T09:40:59.332 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-16T09:40:59.332 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: legacy client fscid: 1 2024-09-16T09:40:59.332 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: 2024-09-16T09:40:59.332 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Filesystem 'foofs' (1) 2024-09-16T09:40:59.332 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: fs_name foofs 2024-09-16T09:40:59.332 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: epoch 5 2024-09-16T09:40:59.332 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-16T09:40:59.333 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: created 2024-09-16T09:35:01.623131+0000 2024-09-16T09:40:59.333 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: modified 2024-09-16T09:35:07.623249+0000 2024-09-16T09:40:59.333 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: tableserver 0 2024-09-16T09:40:59.333 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: root 0 2024-09-16T09:40:59.333 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: session_timeout 60 2024-09-16T09:40:59.333 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: session_autoclose 300 2024-09-16T09:40:59.333 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: max_file_size 1099511627776 2024-09-16T09:40:59.333 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: max_xattr_size 65536 2024-09-16T09:40:59.333 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: required_client_features {} 2024-09-16T09:40:59.333 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: last_failure 0 2024-09-16T09:40:59.333 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: last_failure_osd_epoch 0 2024-09-16T09:40:59.333 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-16T09:40:59.333 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: max_mds 1 2024-09-16T09:40:59.334 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: in 0 2024-09-16T09:40:59.334 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: up {0=14538} 2024-09-16T09:40:59.334 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: failed 2024-09-16T09:40:59.334 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: damaged 2024-09-16T09:40:59.334 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: stopped 2024-09-16T09:40:59.334 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: data_pools [3] 2024-09-16T09:40:59.334 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: metadata_pool 2 2024-09-16T09:40:59.334 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: inline_data disabled 2024-09-16T09:40:59.334 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: balancer 2024-09-16T09:40:59.334 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: bal_rank_mask -1 2024-09-16T09:40:59.334 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: standby_count_wanted 1 2024-09-16T09:40:59.334 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: qdb_cluster leader: 0 members: 2024-09-16T09:40:59.334 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: [mds.foofs.smithi078.hiotnn{0:14538} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.78:6834/245032473,v1:172.21.15.78:6835/245032473] compat {c=[1],r=[1],i=[7ff]}] 2024-09-16T09:40:59.334 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: 2024-09-16T09:40:59.335 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: 2024-09-16T09:40:59.335 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: Standby daemons: 2024-09-16T09:40:59.335 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: 2024-09-16T09:40:59.335 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout: [mds.foofs.smithi088.dbunob{-1:24279} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.88:6832/861752956,v1:172.21.15.88:6833/861752956] compat {c=[1],r=[1],i=[7ff]}] 2024-09-16T09:40:59.335 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: mon.smithi088@-1(???).osd e72 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-16T09:40:59.335 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: mon.smithi088@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-16T09:40:59.335 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: mon.smithi088@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-16T09:40:59.335 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: mon.smithi088@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-16T09:40:59.335 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 ceph-mon[52758]: mon.smithi088@-1(???).paxosservice(auth 1..30) refresh upgraded, format 0 -> 3 2024-09-16T09:40:59.335 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 bash[52740]: 2e552147327ac311d346dc2bc8c6607195094064e66158a49b14e9748bd9b89c 2024-09-16T09:40:59.335 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:58 smithi088 systemd[1]: Started Ceph mon.smithi088 for a70c765e-740d-11ef-bceb-c7b262605968. 2024-09-16T09:40:59.542 INFO:teuthology.orchestra.run.smithi078.stdout: 5 131396 10.45 MB/sec execute 199 sec latency 57.853 ms 2024-09-16T09:40:59.837 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:59 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "mon metadata", "id": "smithi078"} : dispatch 2024-09-16T09:40:59.838 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:59 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "mon metadata", "id": "smithi088"} : dispatch 2024-09-16T09:40:59.838 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:59 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-16T09:40:59.838 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:59 smithi078 ceph-mon[63909]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T09:40:59.838 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:40:59 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:41:00.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:59 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "mon metadata", "id": "smithi078"} : dispatch 2024-09-16T09:41:00.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:59 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "mon metadata", "id": "smithi088"} : dispatch 2024-09-16T09:41:00.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:59 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-16T09:41:00.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:59 smithi088 ceph-mon[52758]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-16T09:41:00.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:40:59 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:41:00.544 INFO:teuthology.orchestra.run.smithi078.stdout: 5 131909 10.47 MB/sec execute 200 sec latency 28.478 ms 2024-09-16T09:41:00.960 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:00 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "mon metadata", "id": "smithi078"} : dispatch 2024-09-16T09:41:00.960 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:00 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "mon metadata", "id": "smithi088"} : dispatch 2024-09-16T09:41:00.960 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:00 smithi078 ceph-mon[63909]: mon.smithi088 calling monitor election 2024-09-16T09:41:00.960 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:00 smithi078 ceph-mon[63909]: mon.smithi078 calling monitor election 2024-09-16T09:41:00.960 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:00 smithi078 ceph-mon[63909]: mon.smithi078 is new leader, mons smithi078,smithi088 in quorum (ranks 0,1) 2024-09-16T09:41:00.960 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:00 smithi078 ceph-mon[63909]: monmap epoch 3 2024-09-16T09:41:00.960 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:00 smithi078 ceph-mon[63909]: fsid a70c765e-740d-11ef-bceb-c7b262605968 2024-09-16T09:41:00.960 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:00 smithi078 ceph-mon[63909]: last_changed 2024-09-16T09:40:59.727186+0000 2024-09-16T09:41:00.960 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:00 smithi078 ceph-mon[63909]: created 2024-09-16T09:26:06.341058+0000 2024-09-16T09:41:00.960 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:00 smithi078 ceph-mon[63909]: min_mon_release 19 (squid) 2024-09-16T09:41:00.960 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:00 smithi078 ceph-mon[63909]: election_strategy: 1 2024-09-16T09:41:00.960 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:00 smithi078 ceph-mon[63909]: 0: [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon.smithi078 2024-09-16T09:41:00.961 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:00 smithi078 ceph-mon[63909]: 1: [v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0] mon.smithi088 2024-09-16T09:41:00.961 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:00 smithi078 ceph-mon[63909]: fsmap foofs:1 {0=foofs.smithi078.hiotnn=up:active} 1 up:standby 2024-09-16T09:41:00.961 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:00 smithi078 ceph-mon[63909]: osdmap e72: 8 total, 8 up, 8 in 2024-09-16T09:41:00.961 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:00 smithi078 ceph-mon[63909]: mgrmap e42: smithi078.rrfxge(active, since 17s), standbys: smithi088.ncxfdz 2024-09-16T09:41:00.961 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:00 smithi078 ceph-mon[63909]: overall HEALTH_OK 2024-09-16T09:41:00.961 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:00 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:00.961 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:00 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:00.961 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:00 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:41:01.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:00 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "mon metadata", "id": "smithi078"} : dispatch 2024-09-16T09:41:01.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:00 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "mon metadata", "id": "smithi088"} : dispatch 2024-09-16T09:41:01.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:00 smithi088 ceph-mon[52758]: mon.smithi088 calling monitor election 2024-09-16T09:41:01.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:00 smithi088 ceph-mon[52758]: mon.smithi078 calling monitor election 2024-09-16T09:41:01.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:00 smithi088 ceph-mon[52758]: mon.smithi078 is new leader, mons smithi078,smithi088 in quorum (ranks 0,1) 2024-09-16T09:41:01.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:00 smithi088 ceph-mon[52758]: monmap epoch 3 2024-09-16T09:41:01.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:00 smithi088 ceph-mon[52758]: fsid a70c765e-740d-11ef-bceb-c7b262605968 2024-09-16T09:41:01.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:00 smithi088 ceph-mon[52758]: last_changed 2024-09-16T09:40:59.727186+0000 2024-09-16T09:41:01.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:00 smithi088 ceph-mon[52758]: created 2024-09-16T09:26:06.341058+0000 2024-09-16T09:41:01.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:00 smithi088 ceph-mon[52758]: min_mon_release 19 (squid) 2024-09-16T09:41:01.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:00 smithi088 ceph-mon[52758]: election_strategy: 1 2024-09-16T09:41:01.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:00 smithi088 ceph-mon[52758]: 0: [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon.smithi078 2024-09-16T09:41:01.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:00 smithi088 ceph-mon[52758]: 1: [v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0] mon.smithi088 2024-09-16T09:41:01.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:00 smithi088 ceph-mon[52758]: fsmap foofs:1 {0=foofs.smithi078.hiotnn=up:active} 1 up:standby 2024-09-16T09:41:01.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:00 smithi088 ceph-mon[52758]: osdmap e72: 8 total, 8 up, 8 in 2024-09-16T09:41:01.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:00 smithi088 ceph-mon[52758]: mgrmap e42: smithi078.rrfxge(active, since 17s), standbys: smithi088.ncxfdz 2024-09-16T09:41:01.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:00 smithi088 ceph-mon[52758]: overall HEALTH_OK 2024-09-16T09:41:01.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:00 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:01.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:00 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:01.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:00 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:41:01.543 INFO:teuthology.orchestra.run.smithi078.stdout: 5 132258 10.45 MB/sec execute 201 sec latency 37.195 ms 2024-09-16T09:41:01.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:01 smithi078 ceph-mon[63909]: pgmap v12: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 5.3 MiB/s wr, 521 op/s 2024-09-16T09:41:02.032 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:01 smithi088 ceph-mon[52758]: pgmap v12: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 5.3 MiB/s wr, 521 op/s 2024-09-16T09:41:02.543 INFO:teuthology.orchestra.run.smithi078.stdout: 5 132739 10.46 MB/sec execute 202 sec latency 32.966 ms 2024-09-16T09:41:03.381 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-16T09:41:03.543 INFO:teuthology.orchestra.run.smithi078.stdout: 5 133119 10.45 MB/sec execute 203 sec latency 61.020 ms 2024-09-16T09:41:03.762 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:03 smithi078 ceph-mon[63909]: pgmap v13: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.2 MiB/s wr, 336 op/s 2024-09-16T09:41:04.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:03 smithi088 ceph-mon[52758]: pgmap v13: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.2 MiB/s wr, 336 op/s 2024-09-16T09:41:04.359 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T09:41:04.359 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (2m) 13s ago 13m 17.6M - 0.27.0 11f11916f8cd 3ebb6832fbcc 2024-09-16T09:41:04.359 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (13m) 13s ago 13m 6405k - 17.2.0 e1d6a67b021e a7184b35ee05 2024-09-16T09:41:04.359 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi088 smithi088 running (11m) 17s ago 11m 6409k - 17.2.0 e1d6a67b021e a9a129cd5d7d 2024-09-16T09:41:04.359 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (2m) 13s ago 11m 45.0M - dad864ee21e9 4a91b97e29b1 2024-09-16T09:41:04.359 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.hiotnn smithi078 running (5m) 13s ago 5m 36.7M - 17.2.0 e1d6a67b021e c43db7767ab9 2024-09-16T09:41:04.359 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi088.dbunob smithi088 running (5m) 17s ago 5m 16.6M - 17.2.0 e1d6a67b021e d235f1bfbb05 2024-09-16T09:41:04.359 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rrfxge smithi078 *:8443,9283,8765 running (98s) 13s ago 14m 488M - 19.3.0-4882-g66e33d6a 261f2dc7b20f 0d25f2f30f1e 2024-09-16T09:41:04.359 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi088.ncxfdz smithi088 *:8443,9283,8765 running (47s) 17s ago 11m 451M - 19.3.0-4882-g66e33d6a 261f2dc7b20f e2d03f701acd 2024-09-16T09:41:04.360 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (30s) 13s ago 14m 34.8M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f f5a3a0fec757 2024-09-16T09:41:04.360 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi088 smithi088 starting - - - 2048M 2024-09-16T09:41:04.360 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.wawosh smithi078 *:2049 running (5m) 13s ago 5m 274M - 4.0 e1d6a67b021e 2544ed9d67f5 2024-09-16T09:41:04.360 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi088.bnccmw smithi088 *:2049 running (5m) 17s ago 5m 81.4M - 4.0 e1d6a67b021e e606adb67049 2024-09-16T09:41:04.360 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (2m) 13s ago 11m 8186k - 1.7.0 72c9c2088986 895ac3ccb2f3 2024-09-16T09:41:04.360 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi088 smithi088 *:9100 running (107s) 17s ago 11m 11.6M - 1.7.0 72c9c2088986 c84a2e50de53 2024-09-16T09:41:04.360 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (10m) 13s ago 10m 482M 1936M 17.2.0 e1d6a67b021e 453389fafdff 2024-09-16T09:41:04.360 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (9m) 13s ago 9m 401M 1936M 17.2.0 e1d6a67b021e d8b72a22a11b 2024-09-16T09:41:04.360 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (9m) 13s ago 9m 279M 1936M 17.2.0 e1d6a67b021e aa521853c9bf 2024-09-16T09:41:04.360 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (9m) 13s ago 9m 441M 1936M 17.2.0 e1d6a67b021e db62172fe90c 2024-09-16T09:41:04.360 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi088 running (8m) 17s ago 8m 446M 2704M 17.2.0 e1d6a67b021e 6f4e80fe127a 2024-09-16T09:41:04.360 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi088 running (8m) 17s ago 8m 491M 2704M 17.2.0 e1d6a67b021e 73e51edb7756 2024-09-16T09:41:04.361 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi088 running (7m) 17s ago 7m 413M 2704M 17.2.0 e1d6a67b021e 7c1459b1a2d9 2024-09-16T09:41:04.361 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi088 running (7m) 17s ago 7m 319M 2704M 17.2.0 e1d6a67b021e 37f619c2ddf9 2024-09-16T09:41:04.361 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (53s) 13s ago 10m 43.9M - 2.51.0 1d3b7f56885b 4c956ead9125 2024-09-16T09:41:04.543 INFO:teuthology.orchestra.run.smithi078.stdout: 5 133511 10.46 MB/sec execute 204 sec latency 54.076 ms 2024-09-16T09:41:04.798 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:04 smithi078 ceph-mon[63909]: from='client.34128 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:41:05.023 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:41:05.024 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-16T09:41:05.024 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:41:05.024 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:41:05.024 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-16T09:41:05.024 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:41:05.024 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:41:05.024 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-16T09:41:05.024 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-16T09:41:05.024 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:41:05.024 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-16T09:41:05.024 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:41:05.025 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:41:05.025 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-16T09:41:05.025 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-16T09:41:05.025 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 4 2024-09-16T09:41:05.025 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-16T09:41:05.025 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:41:05.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:04 smithi088 ceph-mon[52758]: from='client.34128 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:41:05.543 INFO:teuthology.orchestra.run.smithi078.stdout: 5 133972 10.44 MB/sec execute 205 sec latency 36.389 ms 2024-09-16T09:41:05.667 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:41:05.667 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f", 2024-09-16T09:41:05.667 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-16T09:41:05.667 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T09:41:05.667 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-16T09:41:05.667 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-09-16T09:41:05.667 INFO:teuthology.orchestra.run.smithi078.stdout: "mon" 2024-09-16T09:41:05.668 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-16T09:41:05.668 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "4/23 daemons upgraded", 2024-09-16T09:41:05.668 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading mon daemons", 2024-09-16T09:41:05.668 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-16T09:41:05.668 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:41:06.033 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:05 smithi078 ceph-mon[63909]: from='client.34132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:41:06.034 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:05 smithi078 ceph-mon[63909]: pgmap v14: 97 pgs: 97 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.8 MiB/s wr, 574 op/s 2024-09-16T09:41:06.034 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:05 smithi078 ceph-mon[63909]: from='client.44101 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:41:06.034 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:05 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:06.034 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:05 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:06.034 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:05 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/1647366365' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:41:06.034 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:05 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:06.034 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:05 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:06.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:05 smithi088 ceph-mon[52758]: from='client.34132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:41:06.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:05 smithi088 ceph-mon[52758]: pgmap v14: 97 pgs: 97 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.8 MiB/s wr, 574 op/s 2024-09-16T09:41:06.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:05 smithi088 ceph-mon[52758]: from='client.44101 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:41:06.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:05 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:06.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:05 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:06.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:05 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/1647366365' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:41:06.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:05 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:06.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:05 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:06.318 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-16T09:41:06.543 INFO:teuthology.orchestra.run.smithi078.stdout: 5 134283 10.43 MB/sec execute 206 sec latency 51.273 ms 2024-09-16T09:41:07.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:06 smithi078 ceph-mon[63909]: from='client.44109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:41:07.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:06 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/3677510614' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:41:07.068 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:06 smithi088 ceph-mon[52758]: from='client.44109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:41:07.068 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:06 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/3677510614' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:41:07.543 INFO:teuthology.orchestra.run.smithi078.stdout: 5 134672 10.46 MB/sec execute 207 sec latency 42.410 ms 2024-09-16T09:41:07.857 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:07 smithi078 ceph-mon[63909]: pgmap v15: 97 pgs: 97 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.1 MiB/s wr, 406 op/s 2024-09-16T09:41:08.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:07 smithi088 ceph-mon[52758]: pgmap v15: 97 pgs: 97 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.1 MiB/s wr, 406 op/s 2024-09-16T09:41:08.543 INFO:teuthology.orchestra.run.smithi078.stdout: 5 135075 10.47 MB/sec execute 208 sec latency 42.380 ms 2024-09-16T09:41:09.543 INFO:teuthology.orchestra.run.smithi078.stdout: 5 135538 10.49 MB/sec execute 209 sec latency 43.624 ms 2024-09-16T09:41:09.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:09 smithi078 ceph-mon[63909]: pgmap v16: 97 pgs: 97 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.1 MiB/s wr, 406 op/s 2024-09-16T09:41:09.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:09 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:09.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:09 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:09.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:09 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:09.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:09 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:41:09.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:09 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:09.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:09 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:09.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:09 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-16T09:41:09.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:09 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-16T09:41:09.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:09 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:10.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:09 smithi088 ceph-mon[52758]: pgmap v16: 97 pgs: 97 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.1 MiB/s wr, 406 op/s 2024-09-16T09:41:10.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:09 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:10.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:09 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:10.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:09 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:10.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:09 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:41:10.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:09 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:10.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:09 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:10.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:09 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-16T09:41:10.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:09 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-16T09:41:10.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:09 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:10.543 INFO:teuthology.orchestra.run.smithi078.stdout: 5 135773 10.46 MB/sec execute 210 sec latency 35.014 ms 2024-09-16T09:41:10.918 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:10 smithi078 ceph-mon[63909]: Reconfiguring mon.smithi078 (monmap changed)... 2024-09-16T09:41:10.918 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:10 smithi078 ceph-mon[63909]: Reconfiguring daemon mon.smithi078 on smithi078 2024-09-16T09:41:11.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:10 smithi088 ceph-mon[52758]: Reconfiguring mon.smithi078 (monmap changed)... 2024-09-16T09:41:11.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:10 smithi088 ceph-mon[52758]: Reconfiguring daemon mon.smithi078 on smithi078 2024-09-16T09:41:11.543 INFO:teuthology.orchestra.run.smithi078.stdout: 5 136034 10.44 MB/sec execute 211 sec latency 90.114 ms 2024-09-16T09:41:11.963 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:11 smithi078 ceph-mon[63909]: pgmap v17: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.5 MiB/s wr, 634 op/s 2024-09-16T09:41:11.964 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:11 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:11.964 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:11 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:11.964 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:11 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi078.rrfxge", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-16T09:41:11.964 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:11 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "mgr services"} : dispatch 2024-09-16T09:41:11.964 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:11 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:12.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:11 smithi088 ceph-mon[52758]: pgmap v17: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.5 MiB/s wr, 634 op/s 2024-09-16T09:41:12.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:11 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:12.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:11 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:12.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:11 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi078.rrfxge", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-16T09:41:12.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:11 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "mgr services"} : dispatch 2024-09-16T09:41:12.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:11 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:12.543 INFO:teuthology.orchestra.run.smithi078.stdout: 5 136333 10.42 MB/sec execute 212 sec latency 71.681 ms 2024-09-16T09:41:12.974 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:12 smithi078 ceph-mon[63909]: Reconfiguring mgr.smithi078.rrfxge (monmap changed)... 2024-09-16T09:41:12.974 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:12 smithi078 ceph-mon[63909]: Reconfiguring daemon mgr.smithi078.rrfxge on smithi078 2024-09-16T09:41:12.974 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:12 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:41:13.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:12 smithi088 ceph-mon[52758]: Reconfiguring mgr.smithi078.rrfxge (monmap changed)... 2024-09-16T09:41:13.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:12 smithi088 ceph-mon[52758]: Reconfiguring daemon mgr.smithi078.rrfxge on smithi078 2024-09-16T09:41:13.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:12 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:41:13.543 INFO:teuthology.orchestra.run.smithi078.stdout: 5 136686 10.41 MB/sec execute 213 sec latency 84.130 ms 2024-09-16T09:41:13.981 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:13 smithi078 ceph-mon[63909]: pgmap v18: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.9 MiB/s wr, 466 op/s 2024-09-16T09:41:13.982 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:13 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:13.982 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:13 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:13.982 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:13 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-16T09:41:13.982 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:13 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:14.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:13 smithi088 ceph-mon[52758]: pgmap v18: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.9 MiB/s wr, 466 op/s 2024-09-16T09:41:14.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:13 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:14.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:13 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:14.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:13 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-16T09:41:14.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:13 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:14.543 INFO:teuthology.orchestra.run.smithi078.stdout: 5 137150 10.42 MB/sec execute 214 sec latency 45.055 ms 2024-09-16T09:41:14.986 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:14 smithi078 ceph-mon[63909]: Reconfiguring crash.smithi078 (monmap changed)... 2024-09-16T09:41:14.986 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:14 smithi078 ceph-mon[63909]: Reconfiguring daemon crash.smithi078 on smithi078 2024-09-16T09:41:15.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:14 smithi088 ceph-mon[52758]: Reconfiguring crash.smithi078 (monmap changed)... 2024-09-16T09:41:15.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:14 smithi088 ceph-mon[52758]: Reconfiguring daemon crash.smithi078 on smithi078 2024-09-16T09:41:15.543 INFO:teuthology.orchestra.run.smithi078.stdout: 5 137809 10.43 MB/sec execute 215 sec latency 43.532 ms 2024-09-16T09:41:15.992 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:15 smithi078 ceph-mon[63909]: pgmap v19: 97 pgs: 97 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 671 op/s 2024-09-16T09:41:15.993 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:15 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:15.993 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:15 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:15.993 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:15 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-16T09:41:15.993 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:15 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:16.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:15 smithi088 ceph-mon[52758]: pgmap v19: 97 pgs: 97 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 671 op/s 2024-09-16T09:41:16.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:15 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:16.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:15 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:16.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:15 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-16T09:41:16.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:15 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:16.543 INFO:teuthology.orchestra.run.smithi078.stdout: 5 138180 10.46 MB/sec execute 216 sec latency 42.693 ms 2024-09-16T09:41:17.032 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:16 smithi078 ceph-mon[63909]: Reconfiguring osd.0 (monmap changed)... 2024-09-16T09:41:17.033 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:16 smithi078 ceph-mon[63909]: Reconfiguring daemon osd.0 on smithi078 2024-09-16T09:41:17.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:16 smithi088 ceph-mon[52758]: Reconfiguring osd.0 (monmap changed)... 2024-09-16T09:41:17.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:16 smithi088 ceph-mon[52758]: Reconfiguring daemon osd.0 on smithi078 2024-09-16T09:41:17.544 INFO:teuthology.orchestra.run.smithi078.stdout: 5 138481 10.46 MB/sec execute 217 sec latency 67.010 ms 2024-09-16T09:41:17.808 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:17 smithi078 ceph-mon[63909]: pgmap v20: 97 pgs: 97 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.4 MiB/s wr, 432 op/s 2024-09-16T09:41:18.300 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:17 smithi088 ceph-mon[52758]: pgmap v20: 97 pgs: 97 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.4 MiB/s wr, 432 op/s 2024-09-16T09:41:18.543 INFO:teuthology.orchestra.run.smithi078.stdout: 5 139063 10.49 MB/sec execute 218 sec latency 61.437 ms 2024-09-16T09:41:19.335 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:19 smithi078 ceph-mon[63909]: pgmap v21: 97 pgs: 97 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.4 MiB/s wr, 432 op/s 2024-09-16T09:41:19.335 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:19.335 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:19.335 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:19 smithi078 ceph-mon[63909]: Reconfiguring osd.1 (monmap changed)... 2024-09-16T09:41:19.335 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-16T09:41:19.335 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:19.335 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:19 smithi078 ceph-mon[63909]: Reconfiguring daemon osd.1 on smithi078 2024-09-16T09:41:19.543 INFO:teuthology.orchestra.run.smithi078.stdout: 5 139708 10.51 MB/sec execute 219 sec latency 19.344 ms 2024-09-16T09:41:19.550 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:19 smithi088 ceph-mon[52758]: pgmap v21: 97 pgs: 97 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.4 MiB/s wr, 432 op/s 2024-09-16T09:41:19.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:19.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:19.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:19 smithi088 ceph-mon[52758]: Reconfiguring osd.1 (monmap changed)... 2024-09-16T09:41:19.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-16T09:41:19.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:19.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:19 smithi088 ceph-mon[52758]: Reconfiguring daemon osd.1 on smithi078 2024-09-16T09:41:20.543 INFO:teuthology.orchestra.run.smithi078.stdout: 5 140366 10.54 MB/sec execute 220 sec latency 31.984 ms 2024-09-16T09:41:21.544 INFO:teuthology.orchestra.run.smithi078.stdout: 5 140844 10.54 MB/sec execute 221 sec latency 52.258 ms 2024-09-16T09:41:21.851 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:21 smithi078 ceph-mon[63909]: pgmap v22: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 689 op/s 2024-09-16T09:41:21.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:21 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:21.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:21 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:21.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:21 smithi078 ceph-mon[63909]: Reconfiguring osd.2 (monmap changed)... 2024-09-16T09:41:21.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:21 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-16T09:41:21.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:21 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:21.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:21 smithi078 ceph-mon[63909]: Reconfiguring daemon osd.2 on smithi078 2024-09-16T09:41:22.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:21 smithi088 ceph-mon[52758]: pgmap v22: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 689 op/s 2024-09-16T09:41:22.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:21 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:22.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:21 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:22.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:21 smithi088 ceph-mon[52758]: Reconfiguring osd.2 (monmap changed)... 2024-09-16T09:41:22.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:21 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-16T09:41:22.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:21 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:22.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:21 smithi088 ceph-mon[52758]: Reconfiguring daemon osd.2 on smithi078 2024-09-16T09:41:22.543 INFO:teuthology.orchestra.run.smithi078.stdout: 5 141377 10.53 MB/sec execute 222 sec latency 33.436 ms 2024-09-16T09:41:23.543 INFO:teuthology.orchestra.run.smithi078.stdout: 5 141963 10.60 MB/sec execute 223 sec latency 46.608 ms 2024-09-16T09:41:23.889 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:23 smithi078 ceph-mon[63909]: pgmap v23: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.7 MiB/s wr, 461 op/s 2024-09-16T09:41:23.890 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:23 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:23.890 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:23 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:23.890 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:23 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-16T09:41:23.890 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:23 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:24.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:23 smithi088 ceph-mon[52758]: pgmap v23: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.7 MiB/s wr, 461 op/s 2024-09-16T09:41:24.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:23 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:24.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:23 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:24.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:23 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-16T09:41:24.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:23 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:24.544 INFO:teuthology.orchestra.run.smithi078.stdout: 5 142659 10.64 MB/sec execute 224 sec latency 28.270 ms 2024-09-16T09:41:24.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:24 smithi078 ceph-mon[63909]: Reconfiguring osd.3 (monmap changed)... 2024-09-16T09:41:24.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:24 smithi078 ceph-mon[63909]: Reconfiguring daemon osd.3 on smithi078 2024-09-16T09:41:25.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:24 smithi088 ceph-mon[52758]: Reconfiguring osd.3 (monmap changed)... 2024-09-16T09:41:25.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:24 smithi088 ceph-mon[52758]: Reconfiguring daemon osd.3 on smithi078 2024-09-16T09:41:25.544 INFO:teuthology.orchestra.run.smithi078.stdout: 5 143023 10.63 MB/sec execute 225 sec latency 38.757 ms 2024-09-16T09:41:25.909 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:25 smithi078 ceph-mon[63909]: pgmap v24: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 817 op/s 2024-09-16T09:41:25.909 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:25 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:25.910 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:25 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:25.910 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:25 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.hiotnn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-16T09:41:25.910 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:25 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:26.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:25 smithi088 ceph-mon[52758]: pgmap v24: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 817 op/s 2024-09-16T09:41:26.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:25 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:26.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:25 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:26.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:25 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.hiotnn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-16T09:41:26.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:25 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:26.544 INFO:teuthology.orchestra.run.smithi078.stdout: 5 143349 10.61 MB/sec execute 226 sec latency 49.443 ms 2024-09-16T09:41:26.926 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:26 smithi078 ceph-mon[63909]: Reconfiguring mds.foofs.smithi078.hiotnn (monmap changed)... 2024-09-16T09:41:26.926 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:26 smithi078 ceph-mon[63909]: Reconfiguring daemon mds.foofs.smithi078.hiotnn on smithi078 2024-09-16T09:41:27.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:26 smithi088 ceph-mon[52758]: Reconfiguring mds.foofs.smithi078.hiotnn (monmap changed)... 2024-09-16T09:41:27.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:26 smithi088 ceph-mon[52758]: Reconfiguring daemon mds.foofs.smithi078.hiotnn on smithi078 2024-09-16T09:41:27.544 INFO:teuthology.orchestra.run.smithi078.stdout: 5 143617 10.60 MB/sec execute 227 sec latency 70.740 ms 2024-09-16T09:41:27.990 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:27 smithi078 ceph-mon[63909]: pgmap v25: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.1 MiB/s wr, 613 op/s 2024-09-16T09:41:27.990 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:27 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:41:28.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:27 smithi088 ceph-mon[52758]: pgmap v25: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.1 MiB/s wr, 613 op/s 2024-09-16T09:41:28.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:27 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:41:28.544 INFO:teuthology.orchestra.run.smithi078.stdout: 5 143949 10.59 MB/sec execute 228 sec latency 47.149 ms 2024-09-16T09:41:29.003 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:28 smithi078 ceph-mon[63909]: pgmap v26: 97 pgs: 97 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 7.1 MiB/s wr, 663 op/s 2024-09-16T09:41:29.003 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:28 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:29.003 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:28 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:29.003 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:28 smithi078 ceph-mon[63909]: Reconfiguring crash.smithi088 (monmap changed)... 2024-09-16T09:41:29.003 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:28 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi088", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-16T09:41:29.003 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:28 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:29.003 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:28 smithi078 ceph-mon[63909]: Reconfiguring daemon crash.smithi088 on smithi088 2024-09-16T09:41:29.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:28 smithi088 ceph-mon[52758]: pgmap v26: 97 pgs: 97 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 7.1 MiB/s wr, 663 op/s 2024-09-16T09:41:29.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:28 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:29.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:28 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:29.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:28 smithi088 ceph-mon[52758]: Reconfiguring crash.smithi088 (monmap changed)... 2024-09-16T09:41:29.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:28 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi088", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-16T09:41:29.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:28 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:29.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:28 smithi088 ceph-mon[52758]: Reconfiguring daemon crash.smithi088 on smithi088 2024-09-16T09:41:29.544 INFO:teuthology.orchestra.run.smithi078.stdout: 5 144478 10.59 MB/sec execute 229 sec latency 31.992 ms 2024-09-16T09:41:30.544 INFO:teuthology.orchestra.run.smithi078.stdout: 5 144873 10.58 MB/sec execute 230 sec latency 40.735 ms 2024-09-16T09:41:30.770 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:30 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:30.770 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:30 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:30.770 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:30 smithi078 ceph-mon[63909]: Reconfiguring mgr.smithi088.ncxfdz (monmap changed)... 2024-09-16T09:41:30.770 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:30 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi088.ncxfdz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-16T09:41:30.770 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:30 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "mgr services"} : dispatch 2024-09-16T09:41:30.770 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:30 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:30.770 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:30 smithi078 ceph-mon[63909]: Reconfiguring daemon mgr.smithi088.ncxfdz on smithi088 2024-09-16T09:41:30.869 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:30 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:30.869 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:30 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:30.869 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:30 smithi088 ceph-mon[52758]: Reconfiguring mgr.smithi088.ncxfdz (monmap changed)... 2024-09-16T09:41:30.869 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:30 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi088.ncxfdz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-16T09:41:30.869 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:30 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "mgr services"} : dispatch 2024-09-16T09:41:30.869 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:30 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:30.869 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:30 smithi088 ceph-mon[52758]: Reconfiguring daemon mgr.smithi088.ncxfdz on smithi088 2024-09-16T09:41:31.544 INFO:teuthology.orchestra.run.smithi078.stdout: 5 145115 10.57 MB/sec execute 231 sec latency 73.111 ms 2024-09-16T09:41:31.692 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:31 smithi078 ceph-mon[63909]: pgmap v27: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 9.0 MiB/s wr, 880 op/s 2024-09-16T09:41:32.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:31 smithi088 ceph-mon[52758]: pgmap v27: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 9.0 MiB/s wr, 880 op/s 2024-09-16T09:41:32.544 INFO:teuthology.orchestra.run.smithi078.stdout: 5 145508 10.60 MB/sec execute 232 sec latency 42.507 ms 2024-09-16T09:41:32.960 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:32.961 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:32.961 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:32 smithi078 ceph-mon[63909]: Reconfiguring mon.smithi088 (monmap changed)... 2024-09-16T09:41:32.961 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-16T09:41:32.961 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-16T09:41:32.961 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:32.961 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:32 smithi078 ceph-mon[63909]: Reconfiguring daemon mon.smithi088 on smithi088 2024-09-16T09:41:32.961 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:32 smithi078 ceph-mon[63909]: pgmap v28: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.3 MiB/s wr, 623 op/s 2024-09-16T09:41:33.205 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:33.206 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:33.206 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:32 smithi088 ceph-mon[52758]: Reconfiguring mon.smithi088 (monmap changed)... 2024-09-16T09:41:33.206 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-16T09:41:33.206 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-16T09:41:33.206 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:33.206 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:32 smithi088 ceph-mon[52758]: Reconfiguring daemon mon.smithi088 on smithi088 2024-09-16T09:41:33.206 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:32 smithi088 ceph-mon[52758]: pgmap v28: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.3 MiB/s wr, 623 op/s 2024-09-16T09:41:33.544 INFO:teuthology.orchestra.run.smithi078.stdout: 5 145926 10.60 MB/sec execute 233 sec latency 50.403 ms 2024-09-16T09:41:34.286 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:34 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:34.287 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:34 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:34.287 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:34 smithi078 ceph-mon[63909]: Reconfiguring osd.4 (monmap changed)... 2024-09-16T09:41:34.287 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:34 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-16T09:41:34.287 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:34 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:34.287 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:34 smithi078 ceph-mon[63909]: Reconfiguring daemon osd.4 on smithi088 2024-09-16T09:41:34.544 INFO:teuthology.orchestra.run.smithi078.stdout: 5 146219 10.60 MB/sec execute 234 sec latency 49.162 ms 2024-09-16T09:41:34.550 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:34 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:34.550 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:34 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:34.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:34 smithi088 ceph-mon[52758]: Reconfiguring osd.4 (monmap changed)... 2024-09-16T09:41:34.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:34 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-16T09:41:34.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:34 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:34.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:34 smithi088 ceph-mon[52758]: Reconfiguring daemon osd.4 on smithi088 2024-09-16T09:41:35.544 INFO:teuthology.orchestra.run.smithi078.stdout: 5 146392 10.58 MB/sec execute 235 sec latency 70.459 ms 2024-09-16T09:41:35.865 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:35 smithi078 ceph-mon[63909]: pgmap v29: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.7 MiB/s wr, 856 op/s 2024-09-16T09:41:35.866 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:35 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:35.866 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:35 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:35.866 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:35 smithi078 ceph-mon[63909]: Reconfiguring osd.5 (monmap changed)... 2024-09-16T09:41:35.866 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:35 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-16T09:41:35.866 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:35 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:35.866 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:35 smithi078 ceph-mon[63909]: Reconfiguring daemon osd.5 on smithi088 2024-09-16T09:41:36.236 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:35 smithi088 ceph-mon[52758]: pgmap v29: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.7 MiB/s wr, 856 op/s 2024-09-16T09:41:36.236 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:35 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:36.236 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:35 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:36.236 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:35 smithi088 ceph-mon[52758]: Reconfiguring osd.5 (monmap changed)... 2024-09-16T09:41:36.236 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:35 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-16T09:41:36.236 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:35 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:36.236 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:35 smithi088 ceph-mon[52758]: Reconfiguring daemon osd.5 on smithi088 2024-09-16T09:41:36.544 INFO:teuthology.orchestra.run.smithi078.stdout: 5 146549 10.55 MB/sec execute 236 sec latency 65.287 ms 2024-09-16T09:41:36.782 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-16T09:41:37.408 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:37 smithi078 ceph-mon[63909]: pgmap v30: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.2 MiB/s wr, 499 op/s 2024-09-16T09:41:37.409 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:37 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:37.409 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:37 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:37.409 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:37 smithi078 ceph-mon[63909]: Reconfiguring osd.6 (monmap changed)... 2024-09-16T09:41:37.409 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:37 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-16T09:41:37.409 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:37 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:37.409 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:37 smithi078 ceph-mon[63909]: Reconfiguring daemon osd.6 on smithi088 2024-09-16T09:41:37.409 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:37 smithi078 ceph-mon[63909]: from='client.34148 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:41:37.544 INFO:teuthology.orchestra.run.smithi078.stdout: 5 146803 10.52 MB/sec execute 237 sec latency 100.477 ms 2024-09-16T09:41:37.684 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T09:41:37.685 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (3m) 46s ago 13m 17.6M - 0.27.0 11f11916f8cd 3ebb6832fbcc 2024-09-16T09:41:37.685 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (13m) 46s ago 13m 6405k - 17.2.0 e1d6a67b021e a7184b35ee05 2024-09-16T09:41:37.685 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi088 smithi088 running (11m) 32s ago 11m 6409k - 17.2.0 e1d6a67b021e a9a129cd5d7d 2024-09-16T09:41:37.685 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (2m) 46s ago 12m 45.0M - dad864ee21e9 4a91b97e29b1 2024-09-16T09:41:37.685 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.hiotnn smithi078 running (6m) 46s ago 6m 36.7M - 17.2.0 e1d6a67b021e c43db7767ab9 2024-09-16T09:41:37.685 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi088.dbunob smithi088 running (6m) 32s ago 6m 16.8M - 17.2.0 e1d6a67b021e d235f1bfbb05 2024-09-16T09:41:37.685 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rrfxge smithi078 *:8443,9283,8765 running (2m) 46s ago 15m 488M - 19.3.0-4882-g66e33d6a 261f2dc7b20f 0d25f2f30f1e 2024-09-16T09:41:37.686 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi088.ncxfdz smithi088 *:8443,9283,8765 running (80s) 32s ago 11m 452M - 19.3.0-4882-g66e33d6a 261f2dc7b20f e2d03f701acd 2024-09-16T09:41:37.686 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (64s) 46s ago 15m 34.8M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f f5a3a0fec757 2024-09-16T09:41:37.686 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi088 smithi088 running (39s) 32s ago 11m 27.6M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f 2e552147327a 2024-09-16T09:41:37.686 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.wawosh smithi078 *:2049 running (6m) 46s ago 6m 274M - 4.0 e1d6a67b021e 2544ed9d67f5 2024-09-16T09:41:37.686 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi088.bnccmw smithi088 *:2049 running (6m) 32s ago 6m 81.6M - 4.0 e1d6a67b021e e606adb67049 2024-09-16T09:41:37.686 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (2m) 46s ago 12m 8186k - 1.7.0 72c9c2088986 895ac3ccb2f3 2024-09-16T09:41:37.686 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi088 smithi088 *:9100 running (2m) 32s ago 11m 11.7M - 1.7.0 72c9c2088986 c84a2e50de53 2024-09-16T09:41:37.686 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (11m) 46s ago 11m 482M 1936M 17.2.0 e1d6a67b021e 453389fafdff 2024-09-16T09:41:37.686 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (10m) 46s ago 10m 401M 1936M 17.2.0 e1d6a67b021e d8b72a22a11b 2024-09-16T09:41:37.686 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (10m) 46s ago 10m 279M 1936M 17.2.0 e1d6a67b021e aa521853c9bf 2024-09-16T09:41:37.687 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (9m) 46s ago 9m 441M 1936M 17.2.0 e1d6a67b021e db62172fe90c 2024-09-16T09:41:37.687 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi088 running (9m) 32s ago 9m 472M 2704M 17.2.0 e1d6a67b021e 6f4e80fe127a 2024-09-16T09:41:37.687 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi088 running (8m) 32s ago 8m 516M 2704M 17.2.0 e1d6a67b021e 73e51edb7756 2024-09-16T09:41:37.687 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi088 running (8m) 32s ago 8m 423M 2704M 17.2.0 e1d6a67b021e 7c1459b1a2d9 2024-09-16T09:41:37.687 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi088 running (7m) 32s ago 7m 345M 2704M 17.2.0 e1d6a67b021e 37f619c2ddf9 2024-09-16T09:41:37.687 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (87s) 46s ago 11m 43.9M - 2.51.0 1d3b7f56885b 4c956ead9125 2024-09-16T09:41:37.741 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:37 smithi088 ceph-mon[52758]: pgmap v30: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.2 MiB/s wr, 499 op/s 2024-09-16T09:41:37.741 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:37 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:37.741 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:37 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:37.742 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:37 smithi088 ceph-mon[52758]: Reconfiguring osd.6 (monmap changed)... 2024-09-16T09:41:37.742 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:37 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-16T09:41:37.742 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:37 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:37.742 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:37 smithi088 ceph-mon[52758]: Reconfiguring daemon osd.6 on smithi088 2024-09-16T09:41:37.742 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:37 smithi088 ceph-mon[52758]: from='client.34148 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:41:38.242 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:41:38.242 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-16T09:41:38.243 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:41:38.243 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:41:38.243 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-16T09:41:38.243 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:41:38.243 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:41:38.244 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-16T09:41:38.245 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-16T09:41:38.245 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:41:38.245 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-16T09:41:38.245 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:41:38.246 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:41:38.246 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-16T09:41:38.246 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-16T09:41:38.246 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 4 2024-09-16T09:41:38.246 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-16T09:41:38.246 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:41:38.456 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:38 smithi078 ceph-mon[63909]: from='client.44121 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:41:38.457 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:38 smithi078 ceph-mon[63909]: from='client.34156 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:41:38.457 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:38 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:38.457 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:38 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:38.457 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:38 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-16T09:41:38.457 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:38 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:38.457 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:38 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/14110161' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:41:38.528 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:38 smithi088 ceph-mon[52758]: from='client.44121 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:41:38.529 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:38 smithi088 ceph-mon[52758]: from='client.34156 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:41:38.529 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:38 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:38.529 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:38 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:38.529 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:38 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-16T09:41:38.529 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:38 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:38.529 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:38 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/14110161' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:41:38.544 INFO:teuthology.orchestra.run.smithi078.stdout: 5 147012 10.50 MB/sec execute 238 sec latency 58.680 ms 2024-09-16T09:41:38.690 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:41:38.690 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f", 2024-09-16T09:41:38.690 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-16T09:41:38.690 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T09:41:38.691 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-16T09:41:38.691 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-09-16T09:41:38.691 INFO:teuthology.orchestra.run.smithi078.stdout: "mon" 2024-09-16T09:41:38.691 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-16T09:41:38.691 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "4/23 daemons upgraded", 2024-09-16T09:41:38.691 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading mon daemons", 2024-09-16T09:41:38.691 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-16T09:41:38.691 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:41:39.250 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-16T09:41:39.495 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:39 smithi078 ceph-mon[63909]: Reconfiguring osd.7 (monmap changed)... 2024-09-16T09:41:39.496 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:39 smithi078 ceph-mon[63909]: Reconfiguring daemon osd.7 on smithi088 2024-09-16T09:41:39.496 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:39 smithi078 ceph-mon[63909]: pgmap v31: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.6 MiB/s wr, 520 op/s 2024-09-16T09:41:39.496 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:39 smithi078 ceph-mon[63909]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:41:39.496 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:39 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/3098549424' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:41:39.544 INFO:teuthology.orchestra.run.smithi078.stdout: 5 147263 10.47 MB/sec execute 239 sec latency 58.000 ms 2024-09-16T09:41:39.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:39 smithi088 ceph-mon[52758]: Reconfiguring osd.7 (monmap changed)... 2024-09-16T09:41:39.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:39 smithi088 ceph-mon[52758]: Reconfiguring daemon osd.7 on smithi088 2024-09-16T09:41:39.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:39 smithi088 ceph-mon[52758]: pgmap v31: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.6 MiB/s wr, 520 op/s 2024-09-16T09:41:39.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:39 smithi088 ceph-mon[52758]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:41:39.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:39 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/3098549424' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:41:40.544 INFO:teuthology.orchestra.run.smithi078.stdout: 5 147455 10.44 MB/sec execute 240 sec latency 68.962 ms 2024-09-16T09:41:40.776 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:40 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:40.776 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:40 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:40.777 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:40 smithi078 ceph-mon[63909]: Reconfiguring mds.foofs.smithi088.dbunob (monmap changed)... 2024-09-16T09:41:40.777 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:40 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi088.dbunob", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-16T09:41:40.777 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:40 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:40.777 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:40 smithi078 ceph-mon[63909]: Reconfiguring daemon mds.foofs.smithi088.dbunob on smithi088 2024-09-16T09:41:41.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:40 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:41.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:40 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:41.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:40 smithi088 ceph-mon[52758]: Reconfiguring mds.foofs.smithi088.dbunob (monmap changed)... 2024-09-16T09:41:41.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:40 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi088.dbunob", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-16T09:41:41.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:40 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:41.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:40 smithi088 ceph-mon[52758]: Reconfiguring daemon mds.foofs.smithi088.dbunob on smithi088 2024-09-16T09:41:41.544 INFO:teuthology.orchestra.run.smithi078.stdout: 5 147743 10.44 MB/sec execute 241 sec latency 57.785 ms 2024-09-16T09:41:41.813 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:41 smithi078 ceph-mon[63909]: pgmap v32: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.8 MiB/s wr, 596 op/s 2024-09-16T09:41:41.813 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:41.813 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:41.813 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:41:41.813 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:41:41.813 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:41:41.813 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:41.813 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi078"} : dispatch 2024-09-16T09:41:41.813 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi078"}]': finished 2024-09-16T09:41:41.813 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi088"} : dispatch 2024-09-16T09:41:41.813 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi088"}]': finished 2024-09-16T09:41:42.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:41 smithi088 ceph-mon[52758]: pgmap v32: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.8 MiB/s wr, 596 op/s 2024-09-16T09:41:42.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:41 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:42.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:41 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:42.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:41 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:41:42.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:41 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:41:42.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:41 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:41:42.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:41 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:42.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:41 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi078"} : dispatch 2024-09-16T09:41:42.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:41 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi078"}]': finished 2024-09-16T09:41:42.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:41 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi088"} : dispatch 2024-09-16T09:41:42.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:41 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi088"}]': finished 2024-09-16T09:41:42.544 INFO:teuthology.orchestra.run.smithi078.stdout: 5 148166 10.43 MB/sec execute 242 sec latency 72.268 ms 2024-09-16T09:41:42.830 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:42 smithi078 ceph-mon[63909]: Upgrade: Setting container_image for all mon 2024-09-16T09:41:42.830 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:42 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:41:43.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:42 smithi088 ceph-mon[52758]: Upgrade: Setting container_image for all mon 2024-09-16T09:41:43.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:42 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:41:43.544 INFO:teuthology.orchestra.run.smithi078.stdout: 5 148483 10.41 MB/sec execute 243 sec latency 86.052 ms 2024-09-16T09:41:43.853 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:43 smithi078 ceph-mon[63909]: pgmap v33: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 379 op/s 2024-09-16T09:41:43.853 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:43 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:43.853 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:43 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-16T09:41:43.854 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:43 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:44.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:43 smithi088 ceph-mon[52758]: pgmap v33: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 379 op/s 2024-09-16T09:41:44.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:43 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:44.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:43 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-16T09:41:44.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:43 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:44.544 INFO:teuthology.orchestra.run.smithi078.stdout: 5 148650 10.40 MB/sec execute 244 sec latency 63.069 ms 2024-09-16T09:41:44.884 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:44 smithi078 ceph-mon[63909]: Upgrade: Updating crash.smithi078 (1/2) 2024-09-16T09:41:44.884 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:44 smithi078 ceph-mon[63909]: Deploying daemon crash.smithi078 on smithi078 2024-09-16T09:41:45.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:44 smithi088 ceph-mon[52758]: Upgrade: Updating crash.smithi078 (1/2) 2024-09-16T09:41:45.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:44 smithi088 ceph-mon[52758]: Deploying daemon crash.smithi078 on smithi078 2024-09-16T09:41:45.544 INFO:teuthology.orchestra.run.smithi078.stdout: 5 148982 10.42 MB/sec execute 245 sec latency 58.146 ms 2024-09-16T09:41:45.913 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:45 smithi078 ceph-mon[63909]: pgmap v34: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.4 MiB/s wr, 558 op/s 2024-09-16T09:41:46.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:45 smithi088 ceph-mon[52758]: pgmap v34: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.4 MiB/s wr, 558 op/s 2024-09-16T09:41:46.544 INFO:teuthology.orchestra.run.smithi078.stdout: 5 149294 10.40 MB/sec execute 246 sec latency 68.081 ms 2024-09-16T09:41:47.544 INFO:teuthology.orchestra.run.smithi078.stdout: 5 149724 10.42 MB/sec execute 247 sec latency 57.447 ms 2024-09-16T09:41:47.922 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:47 smithi078 ceph-mon[63909]: pgmap v35: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.0 MiB/s wr, 325 op/s 2024-09-16T09:41:48.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:47 smithi088 ceph-mon[52758]: pgmap v35: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.0 MiB/s wr, 325 op/s 2024-09-16T09:41:48.548 INFO:teuthology.orchestra.run.smithi078.stdout: 5 150095 10.42 MB/sec execute 248 sec latency 45.476 ms 2024-09-16T09:41:49.558 INFO:teuthology.orchestra.run.smithi078.stdout: 5 150524 10.42 MB/sec execute 249 sec latency 41.067 ms 2024-09-16T09:41:49.899 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:49 smithi088 ceph-mon[52758]: pgmap v36: 97 pgs: 97 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 352 op/s 2024-09-16T09:41:49.899 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:49 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:49.899 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:49 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:49.951 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:49 smithi078 ceph-mon[63909]: pgmap v36: 97 pgs: 97 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 352 op/s 2024-09-16T09:41:49.952 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:49 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:49.952 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:49 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:50.558 INFO:teuthology.orchestra.run.smithi078.stdout: 5 150837 10.40 MB/sec execute 250 sec latency 78.631 ms 2024-09-16T09:41:51.558 INFO:teuthology.orchestra.run.smithi078.stdout: 5 151110 10.38 MB/sec execute 251 sec latency 46.905 ms 2024-09-16T09:41:52.047 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:51 smithi078 ceph-mon[63909]: pgmap v37: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.8 MiB/s wr, 492 op/s 2024-09-16T09:41:52.048 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:51 smithi078 ceph-mon[63909]: Upgrade: Updating crash.smithi088 (2/2) 2024-09-16T09:41:52.048 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:51 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:52.048 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:51 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi088", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-16T09:41:52.048 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:51 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:52.048 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:51 smithi078 ceph-mon[63909]: Deploying daemon crash.smithi088 on smithi088 2024-09-16T09:41:52.206 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:51 smithi088 ceph-mon[52758]: pgmap v37: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.8 MiB/s wr, 492 op/s 2024-09-16T09:41:52.206 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:51 smithi088 ceph-mon[52758]: Upgrade: Updating crash.smithi088 (2/2) 2024-09-16T09:41:52.206 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:51 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:52.206 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:51 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi088", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-16T09:41:52.206 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:51 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:41:52.206 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:51 smithi088 ceph-mon[52758]: Deploying daemon crash.smithi088 on smithi088 2024-09-16T09:41:52.558 INFO:teuthology.orchestra.run.smithi078.stdout: 5 151578 10.39 MB/sec execute 252 sec latency 45.815 ms 2024-09-16T09:41:53.558 INFO:teuthology.orchestra.run.smithi078.stdout: 5 151942 10.37 MB/sec execute 253 sec latency 67.393 ms 2024-09-16T09:41:53.873 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:53 smithi078 ceph-mon[63909]: pgmap v38: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.7 MiB/s wr, 366 op/s 2024-09-16T09:41:54.039 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:53 smithi088 ceph-mon[52758]: pgmap v38: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.7 MiB/s wr, 366 op/s 2024-09-16T09:41:54.558 INFO:teuthology.orchestra.run.smithi078.stdout: 5 152229 10.37 MB/sec execute 254 sec latency 47.185 ms 2024-09-16T09:41:55.559 INFO:teuthology.orchestra.run.smithi078.stdout: 5 152539 10.38 MB/sec execute 255 sec latency 49.159 ms 2024-09-16T09:41:55.920 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:55 smithi078 ceph-mon[63909]: pgmap v39: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.7 MiB/s wr, 596 op/s 2024-09-16T09:41:55.920 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:55 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:55.920 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:55 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:55.920 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:55 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:41:56.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:55 smithi088 ceph-mon[52758]: pgmap v39: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.7 MiB/s wr, 596 op/s 2024-09-16T09:41:56.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:55 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:56.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:55 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:56.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:55 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:41:56.559 INFO:teuthology.orchestra.run.smithi078.stdout: 5 152901 10.38 MB/sec execute 256 sec latency 49.496 ms 2024-09-16T09:41:57.559 INFO:teuthology.orchestra.run.smithi078.stdout: 5 153146 10.37 MB/sec execute 257 sec latency 71.037 ms 2024-09-16T09:41:57.754 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:57 smithi078 ceph-mon[63909]: pgmap v40: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 417 op/s 2024-09-16T09:41:57.754 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:57 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:41:58.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:57 smithi088 ceph-mon[52758]: pgmap v40: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 417 op/s 2024-09-16T09:41:58.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:57 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:41:58.559 INFO:teuthology.orchestra.run.smithi078.stdout: 5 153318 10.35 MB/sec execute 258 sec latency 79.971 ms 2024-09-16T09:41:59.561 INFO:teuthology.orchestra.run.smithi078.stdout: 5 153466 10.34 MB/sec execute 259 sec latency 95.295 ms 2024-09-16T09:41:59.618 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:59 smithi078 ceph-mon[63909]: pgmap v41: 97 pgs: 97 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.3 MiB/s wr, 527 op/s 2024-09-16T09:41:59.618 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:59 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:59.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:59 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:59.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:59 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:59.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:41:59 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:59.906 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:59 smithi088 ceph-mon[52758]: pgmap v41: 97 pgs: 97 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.3 MiB/s wr, 527 op/s 2024-09-16T09:41:59.906 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:59 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:59.906 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:59 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:59.906 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:59 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:41:59.906 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:41:59 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:00.559 INFO:teuthology.orchestra.run.smithi078.stdout: 5 153606 10.31 MB/sec execute 260 sec latency 82.290 ms 2024-09-16T09:42:01.559 INFO:teuthology.orchestra.run.smithi078.stdout: 5 153912 10.29 MB/sec execute 261 sec latency 63.462 ms 2024-09-16T09:42:01.618 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:01 smithi078 ceph-mon[63909]: pgmap v42: 97 pgs: 97 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.6 MiB/s wr, 568 op/s 2024-09-16T09:42:01.859 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:01 smithi088 ceph-mon[52758]: pgmap v42: 97 pgs: 97 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.6 MiB/s wr, 568 op/s 2024-09-16T09:42:02.559 INFO:teuthology.orchestra.run.smithi078.stdout: 5 154216 10.27 MB/sec execute 262 sec latency 96.660 ms 2024-09-16T09:42:03.559 INFO:teuthology.orchestra.run.smithi078.stdout: 5 154410 10.25 MB/sec execute 263 sec latency 69.444 ms 2024-09-16T09:42:03.770 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:03 smithi078 ceph-mon[63909]: pgmap v43: 97 pgs: 97 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.1 MiB/s wr, 408 op/s 2024-09-16T09:42:03.771 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:03 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:03.771 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:03 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:03.771 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:03 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:03.771 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:03 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:03.771 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:03 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:03.771 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:03 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:04.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:03 smithi088 ceph-mon[52758]: pgmap v43: 97 pgs: 97 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.1 MiB/s wr, 408 op/s 2024-09-16T09:42:04.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:03 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:04.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:03 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:04.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:03 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:04.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:03 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:04.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:03 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:04.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:03 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:04.559 INFO:teuthology.orchestra.run.smithi078.stdout: 5 154659 10.23 MB/sec execute 264 sec latency 46.631 ms 2024-09-16T09:42:05.559 INFO:teuthology.orchestra.run.smithi078.stdout: 5 155048 10.24 MB/sec execute 265 sec latency 72.189 ms 2024-09-16T09:42:05.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:05 smithi078 ceph-mon[63909]: pgmap v44: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.5 MiB/s wr, 551 op/s 2024-09-16T09:42:06.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:05 smithi088 ceph-mon[52758]: pgmap v44: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.5 MiB/s wr, 551 op/s 2024-09-16T09:42:06.559 INFO:teuthology.orchestra.run.smithi078.stdout: 5 155630 10.24 MB/sec execute 266 sec latency 39.060 ms 2024-09-16T09:42:07.559 INFO:teuthology.orchestra.run.smithi078.stdout: 5 156229 10.29 MB/sec execute 267 sec latency 26.148 ms 2024-09-16T09:42:07.781 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:07 smithi078 ceph-mon[63909]: pgmap v45: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.4 MiB/s wr, 320 op/s 2024-09-16T09:42:08.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:07 smithi088 ceph-mon[52758]: pgmap v45: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.4 MiB/s wr, 320 op/s 2024-09-16T09:42:08.559 INFO:teuthology.orchestra.run.smithi078.stdout: 5 156794 10.31 MB/sec execute 268 sec latency 29.497 ms 2024-09-16T09:42:09.559 INFO:teuthology.orchestra.run.smithi078.stdout: 5 157126 10.31 MB/sec execute 269 sec latency 40.631 ms 2024-09-16T09:42:09.728 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-16T09:42:09.784 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:09 smithi078 ceph-mon[63909]: pgmap v46: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.7 MiB/s wr, 437 op/s 2024-09-16T09:42:09.785 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:09 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:09.785 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:09 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:09.785 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:09 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:42:09.785 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:09 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:42:09.785 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:09 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:09.785 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:09 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:10.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:09 smithi088 ceph-mon[52758]: pgmap v46: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.7 MiB/s wr, 437 op/s 2024-09-16T09:42:10.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:09 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:10.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:09 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:10.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:09 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:42:10.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:09 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:42:10.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:09 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:10.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:09 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:10.559 INFO:teuthology.orchestra.run.smithi078.stdout: 5 157386 10.29 MB/sec execute 270 sec latency 61.970 ms 2024-09-16T09:42:10.797 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T09:42:10.797 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (3m) 8s ago 14m 17.6M - 0.27.0 11f11916f8cd 3ebb6832fbcc 2024-09-16T09:42:10.797 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (22s) 8s ago 14m 6899k - 19.3.0-4882-g66e33d6a 261f2dc7b20f 72965213139c 2024-09-16T09:42:10.797 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi088 smithi088 running (16s) 12s ago 12m 6891k - 19.3.0-4882-g66e33d6a 261f2dc7b20f eac0e6855b1b 2024-09-16T09:42:10.797 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (3m) 8s ago 12m 45.8M - dad864ee21e9 4a91b97e29b1 2024-09-16T09:42:10.797 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.hiotnn smithi078 running (7m) 8s ago 7m 37.3M - 17.2.0 e1d6a67b021e c43db7767ab9 2024-09-16T09:42:10.797 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi088.dbunob smithi088 running (7m) 12s ago 7m 17.0M - 17.2.0 e1d6a67b021e d235f1bfbb05 2024-09-16T09:42:10.797 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rrfxge smithi078 *:8443,9283,8765 running (2m) 8s ago 15m 509M - 19.3.0-4882-g66e33d6a 261f2dc7b20f 0d25f2f30f1e 2024-09-16T09:42:10.797 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi088.ncxfdz smithi088 *:8443,9283,8765 running (113s) 12s ago 12m 453M - 19.3.0-4882-g66e33d6a 261f2dc7b20f e2d03f701acd 2024-09-16T09:42:10.797 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (97s) 8s ago 15m 45.9M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f f5a3a0fec757 2024-09-16T09:42:10.798 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi088 smithi088 running (72s) 12s ago 12m 37.7M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f 2e552147327a 2024-09-16T09:42:10.798 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.wawosh smithi078 *:2049 running (6m) 8s ago 6m 279M - 4.0 e1d6a67b021e 2544ed9d67f5 2024-09-16T09:42:10.798 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi088.bnccmw smithi088 *:2049 running (6m) 12s ago 6m 81.6M - 4.0 e1d6a67b021e e606adb67049 2024-09-16T09:42:10.798 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (3m) 8s ago 12m 9945k - 1.7.0 72c9c2088986 895ac3ccb2f3 2024-09-16T09:42:10.798 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi088 smithi088 *:9100 running (2m) 12s ago 12m 10.9M - 1.7.0 72c9c2088986 c84a2e50de53 2024-09-16T09:42:10.798 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (11m) 8s ago 11m 566M 1936M 17.2.0 e1d6a67b021e 453389fafdff 2024-09-16T09:42:10.798 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (11m) 8s ago 11m 472M 1936M 17.2.0 e1d6a67b021e d8b72a22a11b 2024-09-16T09:42:10.798 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (10m) 8s ago 10m 349M 1936M 17.2.0 e1d6a67b021e aa521853c9bf 2024-09-16T09:42:10.798 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (10m) 8s ago 10m 495M 1936M 17.2.0 e1d6a67b021e db62172fe90c 2024-09-16T09:42:10.798 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi088 running (9m) 12s ago 9m 505M 2704M 17.2.0 e1d6a67b021e 6f4e80fe127a 2024-09-16T09:42:10.798 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi088 running (9m) 12s ago 9m 322M 2704M 17.2.0 e1d6a67b021e 73e51edb7756 2024-09-16T09:42:10.799 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi088 running (8m) 12s ago 8m 503M 2704M 17.2.0 e1d6a67b021e 7c1459b1a2d9 2024-09-16T09:42:10.799 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi088 running (8m) 12s ago 8m 398M 2704M 17.2.0 e1d6a67b021e 37f619c2ddf9 2024-09-16T09:42:10.799 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (2m) 8s ago 12m 50.4M - 2.51.0 1d3b7f56885b 4c956ead9125 2024-09-16T09:42:10.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:10 smithi078 ceph-mon[63909]: from='client.34172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:42:10.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:10 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:42:10.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:10 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:42:10.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:10 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:42:10.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:10 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:42:10.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:10 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:10.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:10 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi078"} : dispatch 2024-09-16T09:42:10.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:10 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi078"}]': finished 2024-09-16T09:42:10.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:10 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi088"} : dispatch 2024-09-16T09:42:10.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:10 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi088"}]': finished 2024-09-16T09:42:10.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:10 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-16T09:42:11.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:10 smithi088 ceph-mon[52758]: from='client.34172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:42:11.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:10 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:42:11.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:10 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:42:11.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:10 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:42:11.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:10 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:42:11.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:10 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:11.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:10 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi078"} : dispatch 2024-09-16T09:42:11.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:10 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi078"}]': finished 2024-09-16T09:42:11.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:10 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi088"} : dispatch 2024-09-16T09:42:11.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:10 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi088"}]': finished 2024-09-16T09:42:11.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:10 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-16T09:42:11.381 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:42:11.381 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-16T09:42:11.381 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:42:11.382 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:42:11.382 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-16T09:42:11.382 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:42:11.382 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:42:11.382 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-16T09:42:11.382 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-16T09:42:11.382 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:42:11.382 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-16T09:42:11.382 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:42:11.382 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:42:11.382 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-16T09:42:11.382 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-16T09:42:11.383 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 4 2024-09-16T09:42:11.383 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-16T09:42:11.383 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:42:11.559 INFO:teuthology.orchestra.run.smithi078.stdout: 5 157616 10.27 MB/sec execute 271 sec latency 126.692 ms 2024-09-16T09:42:11.845 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:11 smithi078 ceph-mon[63909]: Upgrade: Setting container_image for all crash 2024-09-16T09:42:11.845 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:11 smithi078 ceph-mon[63909]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-16T09:42:11.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:11 smithi078 ceph-mon[63909]: Upgrade: osd.0 is safe to restart 2024-09-16T09:42:11.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:11 smithi078 ceph-mon[63909]: pgmap v47: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.9 MiB/s wr, 467 op/s 2024-09-16T09:42:11.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:11 smithi078 ceph-mon[63909]: from='client.34176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:42:11.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:11 smithi078 ceph-mon[63909]: from='client.44147 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:42:11.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:11 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/1194946255' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:42:11.919 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:42:11.920 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f", 2024-09-16T09:42:11.920 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-16T09:42:11.920 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T09:42:11.920 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-16T09:42:11.920 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-09-16T09:42:11.920 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-16T09:42:11.920 INFO:teuthology.orchestra.run.smithi078.stdout: "mon" 2024-09-16T09:42:11.920 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-16T09:42:11.920 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "6/23 daemons upgraded", 2024-09-16T09:42:11.920 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-09-16T09:42:11.920 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-16T09:42:11.920 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:42:12.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:11 smithi088 ceph-mon[52758]: Upgrade: Setting container_image for all crash 2024-09-16T09:42:12.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:11 smithi088 ceph-mon[52758]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-16T09:42:12.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:11 smithi088 ceph-mon[52758]: Upgrade: osd.0 is safe to restart 2024-09-16T09:42:12.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:11 smithi088 ceph-mon[52758]: pgmap v47: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.9 MiB/s wr, 467 op/s 2024-09-16T09:42:12.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:11 smithi088 ceph-mon[52758]: from='client.34176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:42:12.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:11 smithi088 ceph-mon[52758]: from='client.44147 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:42:12.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:11 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/1194946255' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:42:12.543 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-16T09:42:12.559 INFO:teuthology.orchestra.run.smithi078.stdout: 5 157881 10.25 MB/sec execute 272 sec latency 50.999 ms 2024-09-16T09:42:12.876 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:12 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:42:12.877 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:12 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:12.877 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:12 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-16T09:42:12.877 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:12 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:42:12.877 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:12 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/2206468602' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:42:13.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:12 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:42:13.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:12 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:13.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:12 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-16T09:42:13.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:12 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:42:13.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:12 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/2206468602' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:42:13.559 INFO:teuthology.orchestra.run.smithi078.stdout: 5 158122 10.24 MB/sec execute 273 sec latency 81.083 ms 2024-09-16T09:42:13.913 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:13 smithi078 ceph-mon[63909]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:42:13.914 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:13 smithi078 ceph-mon[63909]: pgmap v48: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.0 MiB/s wr, 399 op/s 2024-09-16T09:42:13.914 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:13 smithi078 ceph-mon[63909]: Upgrade: Updating osd.0 2024-09-16T09:42:13.914 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:13 smithi078 ceph-mon[63909]: Deploying daemon osd.0 on smithi078 2024-09-16T09:42:14.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:13 smithi088 ceph-mon[52758]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:42:14.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:13 smithi088 ceph-mon[52758]: pgmap v48: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.0 MiB/s wr, 399 op/s 2024-09-16T09:42:14.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:13 smithi088 ceph-mon[52758]: Upgrade: Updating osd.0 2024-09-16T09:42:14.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:13 smithi088 ceph-mon[52758]: Deploying daemon osd.0 on smithi078 2024-09-16T09:42:14.559 INFO:teuthology.orchestra.run.smithi078.stdout: 5 158473 10.24 MB/sec execute 274 sec latency 46.348 ms 2024-09-16T09:42:15.559 INFO:teuthology.orchestra.run.smithi078.stdout: 5 158877 10.23 MB/sec execute 275 sec latency 60.620 ms 2024-09-16T09:42:15.962 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:15 smithi078 ceph-mon[63909]: pgmap v49: 97 pgs: 97 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.0 MiB/s wr, 598 op/s 2024-09-16T09:42:15.962 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:15 smithi078 ceph-mon[63909]: mgrmap e43: smithi078.rrfxge(active, since 92s), standbys: smithi088.ncxfdz 2024-09-16T09:42:16.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:15 smithi088 ceph-mon[52758]: pgmap v49: 97 pgs: 97 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.0 MiB/s wr, 598 op/s 2024-09-16T09:42:16.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:15 smithi088 ceph-mon[52758]: mgrmap e43: smithi078.rrfxge(active, since 92s), standbys: smithi088.ncxfdz 2024-09-16T09:42:16.559 INFO:teuthology.orchestra.run.smithi078.stdout: 5 159194 10.22 MB/sec execute 276 sec latency 48.391 ms 2024-09-16T09:42:16.737 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:16 smithi078 systemd[1]: Stopping Ceph osd.0 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:42:16.988 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:16 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0[34195]: 2024-09-16T09:42:16.944+0000 7f860b363700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T09:42:16.988 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:16 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0[34195]: 2024-09-16T09:42:16.944+0000 7f860b363700 -1 osd.0 72 *** Got signal Terminated *** 2024-09-16T09:42:16.988 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:16 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0[34195]: 2024-09-16T09:42:16.944+0000 7f860b363700 -1 osd.0 72 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T09:42:17.559 INFO:teuthology.orchestra.run.smithi078.stdout: 5 159572 10.24 MB/sec execute 277 sec latency 46.934 ms 2024-09-16T09:42:18.016 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:17 smithi078 ceph-mon[63909]: pgmap v50: 97 pgs: 97 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.6 MiB/s wr, 455 op/s 2024-09-16T09:42:18.016 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:17 smithi078 ceph-mon[63909]: osd.0 marked itself down and dead 2024-09-16T09:42:18.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:17 smithi088 ceph-mon[52758]: pgmap v50: 97 pgs: 97 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.6 MiB/s wr, 455 op/s 2024-09-16T09:42:18.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:17 smithi088 ceph-mon[52758]: osd.0 marked itself down and dead 2024-09-16T09:42:18.559 INFO:teuthology.orchestra.run.smithi078.stdout: 5 159722 10.21 MB/sec execute 278 sec latency 748.170 ms 2024-09-16T09:42:18.561 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:18 smithi078 podman[68290]: 2024-09-16 09:42:18.507771488 +0000 UTC m=+1.732662395 container died 453389fafdffa0b096291d0d90baefeeb0a9a212d6420e2fcb20ba0fa3de2479 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, RELEASE=HEAD, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, ceph=True, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc.) 2024-09-16T09:42:19.066 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:18 smithi078 ceph-mon[63909]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T09:42:19.066 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:18 smithi078 ceph-mon[63909]: osdmap e73: 8 total, 7 up, 8 in 2024-09-16T09:42:19.066 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:18 smithi078 ceph-mon[63909]: osdmap e74: 8 total, 7 up, 8 in 2024-09-16T09:42:19.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:18 smithi088 ceph-mon[52758]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T09:42:19.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:18 smithi088 ceph-mon[52758]: osdmap e73: 8 total, 7 up, 8 in 2024-09-16T09:42:19.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:18 smithi088 ceph-mon[52758]: osdmap e74: 8 total, 7 up, 8 in 2024-09-16T09:42:19.352 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:19 smithi078 podman[68290]: 2024-09-16 09:42:19.261489755 +0000 UTC m=+2.486380662 container cleanup 453389fafdffa0b096291d0d90baefeeb0a9a212d6420e2fcb20ba0fa3de2479 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0, ceph=True, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, distribution-scope=public, name=centos-stream, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, release=754, version=8, RELEASE=HEAD, GIT_CLEAN=True, architecture=x86_64, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8) 2024-09-16T09:42:19.353 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:19 smithi078 bash[68290]: ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0 2024-09-16T09:42:19.560 INFO:teuthology.orchestra.run.smithi078.stdout: 5 160036 10.21 MB/sec execute 279 sec latency 1032.450 ms 2024-09-16T09:42:19.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:19 smithi078 ceph-mon[63909]: pgmap v52: 97 pgs: 23 peering, 3 stale+active+clean, 71 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.2 MiB/s wr, 493 op/s 2024-09-16T09:42:19.855 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:19 smithi078 ceph-mon[63909]: Health check failed: Reduced data availability: 4 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2024-09-16T09:42:19.856 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:19 smithi078 podman[68306]: 2024-09-16 09:42:19.73934597 +0000 UTC m=+1.234504213 container remove 453389fafdffa0b096291d0d90baefeeb0a9a212d6420e2fcb20ba0fa3de2479 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0, release=754, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., version=8, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, name=centos-stream, RELEASE=HEAD, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-16T09:42:20.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:19 smithi088 ceph-mon[52758]: pgmap v52: 97 pgs: 23 peering, 3 stale+active+clean, 71 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.2 MiB/s wr, 493 op/s 2024-09-16T09:42:20.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:19 smithi088 ceph-mon[52758]: Health check failed: Reduced data availability: 4 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2024-09-16T09:42:20.359 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:20 smithi078 podman[68394]: 2024-09-16 09:42:20.015435743 +0000 UTC m=+0.021728062 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:42:20.359 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:20 smithi078 podman[68394]: 2024-09-16 09:42:20.159949801 +0000 UTC m=+0.166242097 container create dc0516445955239f5d30b0ac0b8a4039a391ca6bf5fcfa330b05d9cf0db81da7 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6) 2024-09-16T09:42:20.560 INFO:teuthology.orchestra.run.smithi078.stdout: 5 160356 10.21 MB/sec execute 280 sec latency 55.148 ms 2024-09-16T09:42:20.613 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:20 smithi078 podman[68394]: 2024-09-16 09:42:20.577951551 +0000 UTC m=+0.584243860 container init dc0516445955239f5d30b0ac0b8a4039a391ca6bf5fcfa330b05d9cf0db81da7 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=) 2024-09-16T09:42:20.614 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:20 smithi078 podman[68394]: 2024-09-16 09:42:20.584510322 +0000 UTC m=+0.590802628 container start dc0516445955239f5d30b0ac0b8a4039a391ca6bf5fcfa330b05d9cf0db81da7 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0-deactivate, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-16T09:42:20.875 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:20 smithi078 podman[68394]: 2024-09-16 09:42:20.716854766 +0000 UTC m=+0.723147067 container attach dc0516445955239f5d30b0ac0b8a4039a391ca6bf5fcfa330b05d9cf0db81da7 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-16T09:42:21.153 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:20 smithi078 podman[68394]: 2024-09-16 09:42:20.898682527 +0000 UTC m=+0.904974825 container died dc0516445955239f5d30b0ac0b8a4039a391ca6bf5fcfa330b05d9cf0db81da7 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-16T09:42:21.560 INFO:teuthology.orchestra.run.smithi078.stdout: 5 160636 10.22 MB/sec execute 281 sec latency 73.899 ms 2024-09-16T09:42:21.928 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:21 smithi078 ceph-mon[63909]: pgmap v54: 97 pgs: 36 peering, 61 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.9 MiB/s wr, 604 op/s 2024-09-16T09:42:21.929 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:21 smithi078 podman[68417]: 2024-09-16 09:42:21.890227971 +0000 UTC m=+0.985464128 container remove dc0516445955239f5d30b0ac0b8a4039a391ca6bf5fcfa330b05d9cf0db81da7 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_CLEAN=True, ceph=True) 2024-09-16T09:42:22.180 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:21 smithi078 systemd[1]: ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.0.service: Deactivated successfully. 2024-09-16T09:42:22.180 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:21 smithi078 systemd[1]: Stopped Ceph osd.0 for a70c765e-740d-11ef-bceb-c7b262605968. 2024-09-16T09:42:22.180 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:21 smithi078 systemd[1]: ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.0.service: Consumed 2min 41.426s CPU time. 2024-09-16T09:42:22.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:21 smithi088 ceph-mon[52758]: pgmap v54: 97 pgs: 36 peering, 61 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.9 MiB/s wr, 604 op/s 2024-09-16T09:42:22.439 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:22 smithi078 systemd[1]: Starting Ceph osd.0 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:42:22.560 INFO:teuthology.orchestra.run.smithi078.stdout: 5 161012 10.20 MB/sec execute 282 sec latency 63.447 ms 2024-09-16T09:42:22.693 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:22 smithi078 podman[68513]: 2024-09-16 09:42:22.412481543 +0000 UTC m=+0.021231524 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:42:22.693 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:22 smithi078 podman[68513]: 2024-09-16 09:42:22.585394291 +0000 UTC m=+0.194144275 container create 44eb1bfb792f7bf3db7826afc6be3b56196ca9910d3d4f12c3d0291cc27e534c (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0-activate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6) 2024-09-16T09:42:23.206 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:22 smithi078 podman[68513]: 2024-09-16 09:42:22.978226886 +0000 UTC m=+0.586976867 container init 44eb1bfb792f7bf3db7826afc6be3b56196ca9910d3d4f12c3d0291cc27e534c (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-16T09:42:23.206 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:22 smithi078 podman[68513]: 2024-09-16 09:42:22.986582865 +0000 UTC m=+0.595332847 container start 44eb1bfb792f7bf3db7826afc6be3b56196ca9910d3d4f12c3d0291cc27e534c (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0-activate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-16T09:42:23.206 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:23 smithi078 podman[68513]: 2024-09-16 09:42:23.10569233 +0000 UTC m=+0.714442350 container attach 44eb1bfb792f7bf3db7826afc6be3b56196ca9910d3d4f12c3d0291cc27e534c (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-16T09:42:23.206 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:23 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0-activate[68527]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:42:23.206 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:23 smithi078 bash[68513]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:42:23.206 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:23 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0-activate[68527]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:42:23.206 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:23 smithi078 bash[68513]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:42:23.560 INFO:teuthology.orchestra.run.smithi078.stdout: 5 161330 10.19 MB/sec execute 283 sec latency 83.787 ms 2024-09-16T09:42:24.001 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:23 smithi078 ceph-mon[63909]: pgmap v55: 97 pgs: 36 peering, 61 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 2.9 MiB/s wr, 306 op/s 2024-09-16T09:42:24.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:23 smithi088 ceph-mon[52758]: pgmap v55: 97 pgs: 36 peering, 61 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 2.9 MiB/s wr, 306 op/s 2024-09-16T09:42:24.560 INFO:teuthology.orchestra.run.smithi078.stdout: 5 161590 10.18 MB/sec execute 284 sec latency 98.122 ms 2024-09-16T09:42:24.769 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:24 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0-activate[68527]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-16T09:42:24.770 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:24 smithi078 bash[68513]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-16T09:42:24.770 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:24 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0-activate[68527]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-16T09:42:24.770 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:24 smithi078 bash[68513]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-16T09:42:25.022 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:24 smithi078 ceph-mon[63909]: Health check update: Reduced data availability: 5 pgs inactive, 12 pgs peering (PG_AVAILABILITY) 2024-09-16T09:42:25.022 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:25 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0-activate[68527]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-16T09:42:25.022 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:25 smithi078 bash[68513]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-16T09:42:25.023 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:25 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0-activate[68527]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-16T09:42:25.201 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:24 smithi088 ceph-mon[52758]: Health check update: Reduced data availability: 5 pgs inactive, 12 pgs peering (PG_AVAILABILITY) 2024-09-16T09:42:25.282 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:25 smithi078 bash[68513]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-16T09:42:25.282 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:25 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0-activate[68527]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-16T09:42:25.282 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:25 smithi078 bash[68513]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-16T09:42:25.282 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:25 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0-activate[68527]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-16T09:42:25.282 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:25 smithi078 bash[68513]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-16T09:42:25.282 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:25 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0-activate[68527]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-16T09:42:25.282 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:25 smithi078 bash[68513]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-16T09:42:25.282 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:25 smithi078 podman[68513]: 2024-09-16 09:42:25.068297261 +0000 UTC m=+2.677047251 container died 44eb1bfb792f7bf3db7826afc6be3b56196ca9910d3d4f12c3d0291cc27e534c (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-16T09:42:25.560 INFO:teuthology.orchestra.run.smithi078.stdout: 5 161910 10.18 MB/sec execute 285 sec latency 99.817 ms 2024-09-16T09:42:26.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:25 smithi078 ceph-mon[63909]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.7 MiB/s wr, 576 op/s; 222/1689 objects degraded (13.144%) 2024-09-16T09:42:26.066 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:25 smithi078 ceph-mon[63909]: Health check failed: Degraded data redundancy: 222/1689 objects degraded (13.144%), 29 pgs degraded (PG_DEGRADED) 2024-09-16T09:42:26.066 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:25 smithi078 ceph-mon[63909]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 12 pgs peering) 2024-09-16T09:42:26.216 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:25 smithi088 ceph-mon[52758]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.7 MiB/s wr, 576 op/s; 222/1689 objects degraded (13.144%) 2024-09-16T09:42:26.216 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:25 smithi088 ceph-mon[52758]: Health check failed: Degraded data redundancy: 222/1689 objects degraded (13.144%), 29 pgs degraded (PG_DEGRADED) 2024-09-16T09:42:26.216 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:25 smithi088 ceph-mon[52758]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 12 pgs peering) 2024-09-16T09:42:26.316 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:26 smithi078 podman[68672]: 2024-09-16 09:42:26.104290335 +0000 UTC m=+1.019501160 container remove 44eb1bfb792f7bf3db7826afc6be3b56196ca9910d3d4f12c3d0291cc27e534c (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-16T09:42:26.560 INFO:teuthology.orchestra.run.smithi078.stdout: 5 162303 10.17 MB/sec execute 286 sec latency 65.279 ms 2024-09-16T09:42:26.568 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:26 smithi078 podman[68726]: 2024-09-16 09:42:26.239471084 +0000 UTC m=+0.022105985 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:42:26.568 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:26 smithi078 podman[68726]: 2024-09-16 09:42:26.389034758 +0000 UTC m=+0.171669646 container create 85008b499cb7625e6b7c07fa49c39892daba63b94bdd83a62583dca70529aa14 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.license=GPLv2) 2024-09-16T09:42:26.843 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:26 smithi078 podman[68726]: 2024-09-16 09:42:26.775495954 +0000 UTC m=+0.558130837 container init 85008b499cb7625e6b7c07fa49c39892daba63b94bdd83a62583dca70529aa14 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-16T09:42:26.843 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:26 smithi078 podman[68726]: 2024-09-16 09:42:26.788975221 +0000 UTC m=+0.571610113 container start 85008b499cb7625e6b7c07fa49c39892daba63b94bdd83a62583dca70529aa14 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-16T09:42:26.843 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:26 smithi078 ceph-osd[68743]: -- 172.21.15.78:0/1996959227 <== mon.0 v2:172.21.15.78:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563c653a81e0 con 0x563c64543c00 2024-09-16T09:42:27.114 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:26 smithi078 bash[68726]: 85008b499cb7625e6b7c07fa49c39892daba63b94bdd83a62583dca70529aa14 2024-09-16T09:42:27.114 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:26 smithi078 systemd[1]: Started Ceph osd.0 for a70c765e-740d-11ef-bceb-c7b262605968. 2024-09-16T09:42:27.560 INFO:teuthology.orchestra.run.smithi078.stdout: 5 162570 10.15 MB/sec execute 287 sec latency 56.514 ms 2024-09-16T09:42:27.876 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:27 smithi078 ceph-mon[63909]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.6 MiB/s wr, 567 op/s; 222/1689 objects degraded (13.144%) 2024-09-16T09:42:27.877 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:27 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:27.877 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:27 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:27.877 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:27 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:42:27.877 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:27 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:27.877 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:27 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:42:28.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:27 smithi088 ceph-mon[52758]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.6 MiB/s wr, 567 op/s; 222/1689 objects degraded (13.144%) 2024-09-16T09:42:28.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:27 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:28.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:27 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:28.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:27 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:42:28.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:27 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:28.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:27 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:42:28.560 INFO:teuthology.orchestra.run.smithi078.stdout: 5 162782 10.13 MB/sec execute 288 sec latency 63.266 ms 2024-09-16T09:42:29.178 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:29 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0[68739]: 2024-09-16T09:42:29.095+0000 7fa147615640 -1 Falling back to public interface 2024-09-16T09:42:29.560 INFO:teuthology.orchestra.run.smithi078.stdout: 5 163038 10.13 MB/sec execute 289 sec latency 81.002 ms 2024-09-16T09:42:29.933 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:29 smithi078 ceph-mon[63909]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.2 MiB/s wr, 432 op/s; 220/1659 objects degraded (13.261%) 2024-09-16T09:42:30.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:29 smithi088 ceph-mon[52758]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.2 MiB/s wr, 432 op/s; 220/1659 objects degraded (13.261%) 2024-09-16T09:42:30.560 INFO:teuthology.orchestra.run.smithi078.stdout: 5 163333 10.13 MB/sec execute 290 sec latency 67.783 ms 2024-09-16T09:42:30.958 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:30 smithi078 ceph-mon[63909]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.8 MiB/s wr, 397 op/s; 219/1641 objects degraded (13.346%) 2024-09-16T09:42:30.959 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:30 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 219/1641 objects degraded (13.346%), 29 pgs degraded (PG_DEGRADED) 2024-09-16T09:42:31.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:30 smithi088 ceph-mon[52758]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.8 MiB/s wr, 397 op/s; 219/1641 objects degraded (13.346%) 2024-09-16T09:42:31.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:30 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 219/1641 objects degraded (13.346%), 29 pgs degraded (PG_DEGRADED) 2024-09-16T09:42:31.560 INFO:teuthology.orchestra.run.smithi078.stdout: 5 163630 10.13 MB/sec execute 291 sec latency 68.379 ms 2024-09-16T09:42:32.560 INFO:teuthology.orchestra.run.smithi078.stdout: 5 163853 10.11 MB/sec execute 292 sec latency 63.967 ms 2024-09-16T09:42:33.020 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:32 smithi078 ceph-mon[63909]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.5 MiB/s wr, 368 op/s; 219/1641 objects degraded (13.346%) 2024-09-16T09:42:33.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:32 smithi088 ceph-mon[52758]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.5 MiB/s wr, 368 op/s; 219/1641 objects degraded (13.346%) 2024-09-16T09:42:33.560 INFO:teuthology.orchestra.run.smithi078.stdout: 5 164053 10.10 MB/sec execute 293 sec latency 54.827 ms 2024-09-16T09:42:34.560 INFO:teuthology.orchestra.run.smithi078.stdout: 5 164316 10.10 MB/sec execute 294 sec latency 53.409 ms 2024-09-16T09:42:35.560 INFO:teuthology.orchestra.run.smithi078.stdout: 5 164564 10.07 MB/sec execute 295 sec latency 83.592 ms 2024-09-16T09:42:35.844 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:35 smithi078 ceph-mon[63909]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.4 MiB/s wr, 524 op/s; 226/1677 objects degraded (13.476%) 2024-09-16T09:42:35.845 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:35 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:35.845 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:35 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:35.845 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:35 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:35.845 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:35 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:36.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:35 smithi088 ceph-mon[52758]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.4 MiB/s wr, 524 op/s; 226/1677 objects degraded (13.476%) 2024-09-16T09:42:36.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:35 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:36.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:35 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:36.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:35 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:36.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:35 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:36.561 INFO:teuthology.orchestra.run.smithi078.stdout: 5 164825 10.06 MB/sec execute 296 sec latency 50.900 ms 2024-09-16T09:42:37.380 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:37 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0[68739]: 2024-09-16T09:42:37.321+0000 7fa147615640 -1 osd.0 0 read_superblock omap replica is missing. 2024-09-16T09:42:37.561 INFO:teuthology.orchestra.run.smithi078.stdout: 5 165069 10.04 MB/sec execute 297 sec latency 44.246 ms 2024-09-16T09:42:37.891 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:37 smithi078 ceph-mon[63909]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.5 MiB/s wr, 344 op/s; 226/1677 objects degraded (13.476%) 2024-09-16T09:42:38.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:37 smithi088 ceph-mon[52758]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.5 MiB/s wr, 344 op/s; 226/1677 objects degraded (13.476%) 2024-09-16T09:42:38.394 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:38 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0[68739]: 2024-09-16T09:42:38.374+0000 7fa147615640 -1 osd.0 72 log_to_monitors true 2024-09-16T09:42:38.561 INFO:teuthology.orchestra.run.smithi078.stdout: 5 165393 10.05 MB/sec execute 298 sec latency 41.502 ms 2024-09-16T09:42:38.928 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:38 smithi078 ceph-mon[63909]: from='osd.0 [v2:172.21.15.78:6802/2768231050,v1:172.21.15.78:6803/2768231050]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-16T09:42:39.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:38 smithi088 ceph-mon[52758]: from='osd.0 [v2:172.21.15.78:6802/2768231050,v1:172.21.15.78:6803/2768231050]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-16T09:42:39.440 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:42:39 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0[68739]: 2024-09-16T09:42:39.313+0000 7fa13b3b7640 -1 osd.0 72 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T09:42:39.562 INFO:teuthology.orchestra.run.smithi078.stdout: 5 165699 10.03 MB/sec execute 299 sec latency 111.587 ms 2024-09-16T09:42:39.950 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:39 smithi078 ceph-mon[63909]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.8 MiB/s wr, 494 op/s; 208/1614 objects degraded (12.887%) 2024-09-16T09:42:39.950 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:39 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 208/1614 objects degraded (12.887%), 29 pgs degraded (PG_DEGRADED) 2024-09-16T09:42:39.950 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:39 smithi078 ceph-mon[63909]: from='osd.0 [v2:172.21.15.78:6802/2768231050,v1:172.21.15.78:6803/2768231050]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-16T09:42:39.950 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:39 smithi078 ceph-mon[63909]: osdmap e75: 8 total, 7 up, 8 in 2024-09-16T09:42:39.950 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:39 smithi078 ceph-mon[63909]: from='osd.0 [v2:172.21.15.78:6802/2768231050,v1:172.21.15.78:6803/2768231050]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]} : dispatch 2024-09-16T09:42:39.950 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:39 smithi078 ceph-mon[63909]: from='osd.0 [v2:172.21.15.78:6802/2768231050,v1:172.21.15.78:6803/2768231050]' entity='osd.0' 2024-09-16T09:42:40.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:39 smithi088 ceph-mon[52758]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.8 MiB/s wr, 494 op/s; 208/1614 objects degraded (12.887%) 2024-09-16T09:42:40.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:39 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 208/1614 objects degraded (12.887%), 29 pgs degraded (PG_DEGRADED) 2024-09-16T09:42:40.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:39 smithi088 ceph-mon[52758]: from='osd.0 [v2:172.21.15.78:6802/2768231050,v1:172.21.15.78:6803/2768231050]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-16T09:42:40.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:39 smithi088 ceph-mon[52758]: osdmap e75: 8 total, 7 up, 8 in 2024-09-16T09:42:40.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:39 smithi088 ceph-mon[52758]: from='osd.0 [v2:172.21.15.78:6802/2768231050,v1:172.21.15.78:6803/2768231050]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]} : dispatch 2024-09-16T09:42:40.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:39 smithi088 ceph-mon[52758]: from='osd.0 [v2:172.21.15.78:6802/2768231050,v1:172.21.15.78:6803/2768231050]' entity='osd.0' 2024-09-16T09:42:40.562 INFO:teuthology.orchestra.run.smithi078.stdout: 5 165997 10.01 MB/sec execute 300 sec latency 272.027 ms 2024-09-16T09:42:41.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:40 smithi088 ceph-mon[52758]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T09:42:41.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:40 smithi088 ceph-mon[52758]: osd.0 [v2:172.21.15.78:6802/2768231050,v1:172.21.15.78:6803/2768231050] boot 2024-09-16T09:42:41.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:40 smithi088 ceph-mon[52758]: osdmap e76: 8 total, 8 up, 8 in 2024-09-16T09:42:41.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:40 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-16T09:42:41.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:40 smithi078 ceph-mon[63909]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T09:42:41.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:40 smithi078 ceph-mon[63909]: osd.0 [v2:172.21.15.78:6802/2768231050,v1:172.21.15.78:6803/2768231050] boot 2024-09-16T09:42:41.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:40 smithi078 ceph-mon[63909]: osdmap e76: 8 total, 8 up, 8 in 2024-09-16T09:42:41.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:40 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-16T09:42:41.562 INFO:teuthology.orchestra.run.smithi078.stdout: 5 166031 9.99 MB/sec execute 301 sec latency 1272.056 ms 2024-09-16T09:42:42.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:41 smithi088 ceph-mon[52758]: pgmap v65: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 386 op/s; 205/1596 objects degraded (12.845%) 2024-09-16T09:42:42.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:41 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:42.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:41 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:42.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:41 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:42:42.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:41 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:42:42.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:41 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:42.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:41 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:42.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:41 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:42:42.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:41 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:42:42.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:41 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:42:42.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:41 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:42:42.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:41 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-16T09:42:42.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:41 smithi088 ceph-mon[52758]: osdmap e77: 8 total, 8 up, 8 in 2024-09-16T09:42:42.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:41 smithi078 ceph-mon[63909]: pgmap v65: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 386 op/s; 205/1596 objects degraded (12.845%) 2024-09-16T09:42:42.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:42.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:42.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:42:42.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:42:42.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:42.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:42.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:42:42.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:42:42.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:42:42.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:42:42.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:41 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-16T09:42:42.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:41 smithi078 ceph-mon[63909]: osdmap e77: 8 total, 8 up, 8 in 2024-09-16T09:42:42.562 INFO:teuthology.orchestra.run.smithi078.stdout: 5 166107 9.96 MB/sec execute 302 sec latency 2122.998 ms 2024-09-16T09:42:42.808 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:42 smithi078 ceph-mon[63909]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-16T09:42:42.808 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:42 smithi078 ceph-mon[63909]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-16T09:42:42.809 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:42 smithi078 ceph-mon[63909]: osdmap e78: 8 total, 8 up, 8 in 2024-09-16T09:42:42.809 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:42 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:42.809 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:42 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:42:43.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:42 smithi088 ceph-mon[52758]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-16T09:42:43.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:42 smithi088 ceph-mon[52758]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-16T09:42:43.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:42 smithi088 ceph-mon[52758]: osdmap e78: 8 total, 8 up, 8 in 2024-09-16T09:42:43.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:42 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:43.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:42 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:42:43.204 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-16T09:42:43.562 INFO:teuthology.orchestra.run.smithi078.stdout: 5 166421 9.96 MB/sec execute 303 sec latency 37.189 ms 2024-09-16T09:42:43.837 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:43 smithi078 ceph-mon[63909]: pgmap v68: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.2 MiB/s wr, 332 op/s; 205/1596 objects degraded (12.845%) 2024-09-16T09:42:44.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:43 smithi088 ceph-mon[52758]: pgmap v68: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.2 MiB/s wr, 332 op/s; 205/1596 objects degraded (12.845%) 2024-09-16T09:42:44.367 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T09:42:44.368 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (4m) 9s ago 15m 17.6M - 0.27.0 11f11916f8cd 3ebb6832fbcc 2024-09-16T09:42:44.368 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (55s) 9s ago 15m 6899k - 19.3.0-4882-g66e33d6a 261f2dc7b20f 72965213139c 2024-09-16T09:42:44.368 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi088 smithi088 running (50s) 45s ago 13m 6891k - 19.3.0-4882-g66e33d6a 261f2dc7b20f eac0e6855b1b 2024-09-16T09:42:44.368 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (4m) 9s ago 13m 44.8M - dad864ee21e9 4a91b97e29b1 2024-09-16T09:42:44.368 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.hiotnn smithi078 running (7m) 9s ago 7m 37.4M - 17.2.0 e1d6a67b021e c43db7767ab9 2024-09-16T09:42:44.368 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi088.dbunob smithi088 running (7m) 45s ago 7m 17.0M - 17.2.0 e1d6a67b021e d235f1bfbb05 2024-09-16T09:42:44.368 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rrfxge smithi078 *:8443,9283,8765 running (3m) 9s ago 16m 511M - 19.3.0-4882-g66e33d6a 261f2dc7b20f 0d25f2f30f1e 2024-09-16T09:42:44.368 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi088.ncxfdz smithi088 *:8443,9283,8765 running (2m) 45s ago 13m 453M - 19.3.0-4882-g66e33d6a 261f2dc7b20f e2d03f701acd 2024-09-16T09:42:44.368 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (2m) 9s ago 16m 49.3M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f f5a3a0fec757 2024-09-16T09:42:44.368 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi088 smithi088 running (106s) 45s ago 12m 37.7M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f 2e552147327a 2024-09-16T09:42:44.368 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.wawosh smithi078 *:2049 running (7m) 9s ago 7m 280M - 4.0 e1d6a67b021e 2544ed9d67f5 2024-09-16T09:42:44.368 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi088.bnccmw smithi088 *:2049 running (7m) 45s ago 7m 81.6M - 4.0 e1d6a67b021e e606adb67049 2024-09-16T09:42:44.369 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (3m) 9s ago 13m 9.83M - 1.7.0 72c9c2088986 895ac3ccb2f3 2024-09-16T09:42:44.369 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi088 smithi088 *:9100 running (3m) 45s ago 12m 10.9M - 1.7.0 72c9c2088986 c84a2e50de53 2024-09-16T09:42:44.369 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (18s) 9s ago 12m 12.1M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 85008b499cb7 2024-09-16T09:42:44.369 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (11m) 9s ago 11m 484M 1936M 17.2.0 e1d6a67b021e d8b72a22a11b 2024-09-16T09:42:44.369 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (11m) 9s ago 11m 379M 1936M 17.2.0 e1d6a67b021e aa521853c9bf 2024-09-16T09:42:44.369 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (10m) 9s ago 10m 516M 1936M 17.2.0 e1d6a67b021e db62172fe90c 2024-09-16T09:42:44.369 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi088 running (10m) 45s ago 10m 505M 2704M 17.2.0 e1d6a67b021e 6f4e80fe127a 2024-09-16T09:42:44.369 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi088 running (9m) 45s ago 9m 322M 2704M 17.2.0 e1d6a67b021e 73e51edb7756 2024-09-16T09:42:44.369 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi088 running (9m) 45s ago 9m 503M 2704M 17.2.0 e1d6a67b021e 7c1459b1a2d9 2024-09-16T09:42:44.369 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi088 running (9m) 45s ago 9m 398M 2704M 17.2.0 e1d6a67b021e 37f619c2ddf9 2024-09-16T09:42:44.369 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (2m) 9s ago 12m 50.5M - 2.51.0 1d3b7f56885b 4c956ead9125 2024-09-16T09:42:44.562 INFO:teuthology.orchestra.run.smithi078.stdout: 5 166744 9.96 MB/sec execute 304 sec latency 240.332 ms 2024-09-16T09:42:44.850 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:44 smithi078 ceph-mon[63909]: from='client.44159 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:42:44.851 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:44 smithi078 ceph-mon[63909]: from='client.44163 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:42:44.851 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:44 smithi078 ceph-mon[63909]: osdmap e79: 8 total, 8 up, 8 in 2024-09-16T09:42:44.851 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:44 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 205/1596 objects degraded (12.845%), 29 pgs degraded (PG_DEGRADED) 2024-09-16T09:42:44.851 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:44 smithi078 ceph-mon[63909]: osdmap e80: 8 total, 8 up, 8 in 2024-09-16T09:42:45.063 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:42:45.063 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-16T09:42:45.063 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:42:45.063 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:42:45.063 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-16T09:42:45.063 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:42:45.064 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:42:45.064 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-16T09:42:45.064 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-16T09:42:45.064 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 1 2024-09-16T09:42:45.064 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:42:45.064 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-16T09:42:45.064 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:42:45.064 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:42:45.064 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-16T09:42:45.064 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-16T09:42:45.064 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 5 2024-09-16T09:42:45.064 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-16T09:42:45.064 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:42:45.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:44 smithi088 ceph-mon[52758]: from='client.44159 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:42:45.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:44 smithi088 ceph-mon[52758]: from='client.44163 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:42:45.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:44 smithi088 ceph-mon[52758]: osdmap e79: 8 total, 8 up, 8 in 2024-09-16T09:42:45.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:44 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 205/1596 objects degraded (12.845%), 29 pgs degraded (PG_DEGRADED) 2024-09-16T09:42:45.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:44 smithi088 ceph-mon[52758]: osdmap e80: 8 total, 8 up, 8 in 2024-09-16T09:42:45.563 INFO:teuthology.orchestra.run.smithi078.stdout: 5 167072 9.95 MB/sec execute 305 sec latency 600.644 ms 2024-09-16T09:42:45.608 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:42:45.609 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f", 2024-09-16T09:42:45.609 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-16T09:42:45.609 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T09:42:45.609 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-16T09:42:45.609 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-09-16T09:42:45.609 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-16T09:42:45.609 INFO:teuthology.orchestra.run.smithi078.stdout: "mon" 2024-09-16T09:42:45.609 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-16T09:42:45.609 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "7/23 daemons upgraded", 2024-09-16T09:42:45.609 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-09-16T09:42:45.609 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-16T09:42:45.610 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:42:45.867 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:45 smithi078 ceph-mon[63909]: pgmap v71: 97 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 88 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.1 MiB/s wr, 386 op/s; 176/1494 objects degraded (11.780%); 28 KiB/s, 22 objects/s recovering 2024-09-16T09:42:45.868 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:45 smithi078 ceph-mon[63909]: from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:42:45.868 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:45 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/3989811517' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:42:45.868 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:45 smithi078 ceph-mon[63909]: osdmap e81: 8 total, 8 up, 8 in 2024-09-16T09:42:46.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:45 smithi088 ceph-mon[52758]: pgmap v71: 97 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 88 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.1 MiB/s wr, 386 op/s; 176/1494 objects degraded (11.780%); 28 KiB/s, 22 objects/s recovering 2024-09-16T09:42:46.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:45 smithi088 ceph-mon[52758]: from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:42:46.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:45 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/3989811517' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:42:46.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:45 smithi088 ceph-mon[52758]: osdmap e81: 8 total, 8 up, 8 in 2024-09-16T09:42:46.363 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_WARN Degraded data redundancy: 176/1494 objects degraded (11.780%), 8 pgs degraded 2024-09-16T09:42:46.363 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 176/1494 objects degraded (11.780%), 8 pgs degraded 2024-09-16T09:42:46.363 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1 is active+recovery_wait+undersized+degraded+remapped, acting [3,4] 2024-09-16T09:42:46.363 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.6 is active+recovery_wait+undersized+degraded+remapped, acting [1,4] 2024-09-16T09:42:46.363 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.7 is active+recovery_wait+undersized+degraded+remapped, acting [3,7] 2024-09-16T09:42:46.364 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.f is active+recovery_wait+undersized+degraded+remapped, acting [7,4] 2024-09-16T09:42:46.364 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.12 is active+recovery_wait+undersized+degraded+remapped, acting [3,7] 2024-09-16T09:42:46.364 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.18 is active+recovery_wait+undersized+degraded+remapped, acting [3,1] 2024-09-16T09:42:46.364 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1b is active+recovery_wait+undersized+degraded+remapped, acting [4,7] 2024-09-16T09:42:46.364 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1f is active+recovery_wait+undersized+degraded+remapped, acting [2,5] 2024-09-16T09:42:46.563 INFO:teuthology.orchestra.run.smithi078.stdout: 5 167365 9.96 MB/sec execute 306 sec latency 281.726 ms 2024-09-16T09:42:46.886 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:46 smithi078 ceph-mon[63909]: from='client.34216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:42:46.886 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:46 smithi078 ceph-mon[63909]: osdmap e82: 8 total, 8 up, 8 in 2024-09-16T09:42:46.887 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:46 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/3801172852' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:42:47.300 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:46 smithi088 ceph-mon[52758]: from='client.34216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:42:47.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:46 smithi088 ceph-mon[52758]: osdmap e82: 8 total, 8 up, 8 in 2024-09-16T09:42:47.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:46 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/3801172852' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:42:47.563 INFO:teuthology.orchestra.run.smithi078.stdout: 5 167624 9.96 MB/sec execute 307 sec latency 47.796 ms 2024-09-16T09:42:48.051 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:47 smithi078 ceph-mon[63909]: pgmap v74: 97 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 88 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.1 MiB/s wr, 386 op/s; 176/1494 objects degraded (11.780%); 28 KiB/s, 22 objects/s recovering 2024-09-16T09:42:48.052 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:47 smithi078 ceph-mon[63909]: osdmap e83: 8 total, 8 up, 8 in 2024-09-16T09:42:48.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:47 smithi088 ceph-mon[52758]: pgmap v74: 97 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 88 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.1 MiB/s wr, 386 op/s; 176/1494 objects degraded (11.780%); 28 KiB/s, 22 objects/s recovering 2024-09-16T09:42:48.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:47 smithi088 ceph-mon[52758]: osdmap e83: 8 total, 8 up, 8 in 2024-09-16T09:42:48.563 INFO:teuthology.orchestra.run.smithi078.stdout: 5 167891 9.95 MB/sec execute 308 sec latency 62.328 ms 2024-09-16T09:42:49.563 INFO:teuthology.orchestra.run.smithi078.stdout: 5 168211 9.94 MB/sec execute 309 sec latency 51.552 ms 2024-09-16T09:42:49.838 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:49 smithi078 ceph-mon[63909]: pgmap v77: 97 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 3.4 MiB/s wr, 232 op/s; 132/1593 objects degraded (8.286%); 0 B/s, 10 objects/s recovering 2024-09-16T09:42:49.838 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:49 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 132/1593 objects degraded (8.286%), 6 pgs degraded (PG_DEGRADED) 2024-09-16T09:42:50.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:49 smithi088 ceph-mon[52758]: pgmap v77: 97 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 3.4 MiB/s wr, 232 op/s; 132/1593 objects degraded (8.286%); 0 B/s, 10 objects/s recovering 2024-09-16T09:42:50.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:49 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 132/1593 objects degraded (8.286%), 6 pgs degraded (PG_DEGRADED) 2024-09-16T09:42:50.563 INFO:teuthology.orchestra.run.smithi078.stdout: 5 168533 9.92 MB/sec execute 310 sec latency 53.345 ms 2024-09-16T09:42:51.122 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:50 smithi078 ceph-mon[63909]: pgmap v78: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 4.4 MiB/s wr, 366 op/s; 0 B/s, 30 objects/s recovering 2024-09-16T09:42:51.123 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:50 smithi078 ceph-mon[63909]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 132/1593 objects degraded (8.286%), 6 pgs degraded) 2024-09-16T09:42:51.123 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:50 smithi078 ceph-mon[63909]: Cluster is now healthy 2024-09-16T09:42:51.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:50 smithi088 ceph-mon[52758]: pgmap v78: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 4.4 MiB/s wr, 366 op/s; 0 B/s, 30 objects/s recovering 2024-09-16T09:42:51.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:50 smithi088 ceph-mon[52758]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 132/1593 objects degraded (8.286%), 6 pgs degraded) 2024-09-16T09:42:51.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:50 smithi088 ceph-mon[52758]: Cluster is now healthy 2024-09-16T09:42:51.563 INFO:teuthology.orchestra.run.smithi078.stdout: 5 168856 9.92 MB/sec execute 311 sec latency 38.558 ms 2024-09-16T09:42:52.563 INFO:teuthology.orchestra.run.smithi078.stdout: 5 169267 9.93 MB/sec execute 312 sec latency 63.004 ms 2024-09-16T09:42:53.177 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:53 smithi078 ceph-mon[63909]: pgmap v79: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.8 KiB/s rd, 3.8 MiB/s wr, 312 op/s; 0 B/s, 25 objects/s recovering 2024-09-16T09:42:53.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:53 smithi088 ceph-mon[52758]: pgmap v79: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.8 KiB/s rd, 3.8 MiB/s wr, 312 op/s; 0 B/s, 25 objects/s recovering 2024-09-16T09:42:53.563 INFO:teuthology.orchestra.run.smithi078.stdout: 5 169724 9.92 MB/sec execute 313 sec latency 47.188 ms 2024-09-16T09:42:54.563 INFO:teuthology.orchestra.run.smithi078.stdout: 5 169966 9.91 MB/sec execute 314 sec latency 69.686 ms 2024-09-16T09:42:55.231 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:55 smithi078 ceph-mon[63909]: pgmap v80: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.0 MiB/s wr, 590 op/s; 0 B/s, 21 objects/s recovering 2024-09-16T09:42:55.300 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:55 smithi088 ceph-mon[52758]: pgmap v80: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.0 MiB/s wr, 590 op/s; 0 B/s, 21 objects/s recovering 2024-09-16T09:42:55.563 INFO:teuthology.orchestra.run.smithi078.stdout: 5 170321 9.92 MB/sec execute 315 sec latency 47.769 ms 2024-09-16T09:42:56.563 INFO:teuthology.orchestra.run.smithi078.stdout: 5 170618 9.91 MB/sec execute 316 sec latency 84.085 ms 2024-09-16T09:42:57.043 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:57 smithi078 ceph-mon[63909]: pgmap v81: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.9 MiB/s wr, 481 op/s; 0 B/s, 17 objects/s recovering 2024-09-16T09:42:57.044 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:57 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-16T09:42:57.044 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:57 smithi078 ceph-mon[63909]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-16T09:42:57.044 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:57 smithi078 ceph-mon[63909]: Upgrade: osd.1 is safe to restart 2024-09-16T09:42:57.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:57 smithi088 ceph-mon[52758]: pgmap v81: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.9 MiB/s wr, 481 op/s; 0 B/s, 17 objects/s recovering 2024-09-16T09:42:57.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:57 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-16T09:42:57.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:57 smithi088 ceph-mon[52758]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-16T09:42:57.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:57 smithi088 ceph-mon[52758]: Upgrade: osd.1 is safe to restart 2024-09-16T09:42:57.563 INFO:teuthology.orchestra.run.smithi078.stdout: 5 170952 9.92 MB/sec execute 317 sec latency 108.667 ms 2024-09-16T09:42:58.320 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:58 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:58.320 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:58 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:42:58.320 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:58 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:58.563 INFO:teuthology.orchestra.run.smithi078.stdout: 5 171240 9.93 MB/sec execute 318 sec latency 38.539 ms 2024-09-16T09:42:58.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:58 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:58.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:58 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:42:58.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:58 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:59.563 INFO:teuthology.orchestra.run.smithi078.stdout: 5 171535 9.92 MB/sec execute 319 sec latency 47.135 ms 2024-09-16T09:42:59.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:59 smithi078 ceph-mon[63909]: pgmap v82: 97 pgs: 97 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 6.0 MiB/s wr, 581 op/s; 0 B/s, 16 objects/s recovering 2024-09-16T09:42:59.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:59 smithi078 ceph-mon[63909]: Upgrade: Updating osd.1 2024-09-16T09:42:59.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:59 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:59.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:59 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-16T09:42:59.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:59 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:42:59.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:42:59 smithi078 ceph-mon[63909]: Deploying daemon osd.1 on smithi078 2024-09-16T09:42:59.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:59 smithi088 ceph-mon[52758]: pgmap v82: 97 pgs: 97 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 6.0 MiB/s wr, 581 op/s; 0 B/s, 16 objects/s recovering 2024-09-16T09:42:59.805 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:59 smithi088 ceph-mon[52758]: Upgrade: Updating osd.1 2024-09-16T09:42:59.806 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:59 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:42:59.806 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:59 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-16T09:42:59.806 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:59 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:42:59.806 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:42:59 smithi088 ceph-mon[52758]: Deploying daemon osd.1 on smithi078 2024-09-16T09:43:00.563 INFO:teuthology.orchestra.run.smithi078.stdout: 5 171962 9.91 MB/sec execute 320 sec latency 30.314 ms 2024-09-16T09:43:01.563 INFO:teuthology.orchestra.run.smithi078.stdout: 5 172341 9.91 MB/sec execute 321 sec latency 67.002 ms 2024-09-16T09:43:01.627 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:01 smithi078 ceph-mon[63909]: pgmap v83: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.1 MiB/s wr, 510 op/s; 0 B/s, 10 objects/s recovering 2024-09-16T09:43:01.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:01 smithi088 ceph-mon[52758]: pgmap v83: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.1 MiB/s wr, 510 op/s; 0 B/s, 10 objects/s recovering 2024-09-16T09:43:02.564 INFO:teuthology.orchestra.run.smithi078.stdout: 5 172838 9.92 MB/sec execute 322 sec latency 48.486 ms 2024-09-16T09:43:03.390 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:03 smithi078 systemd[1]: Stopping Ceph osd.1 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:43:03.564 INFO:teuthology.orchestra.run.smithi078.stdout: 5 173438 9.93 MB/sec execute 323 sec latency 32.094 ms 2024-09-16T09:43:03.640 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:03 smithi078 ceph-mon[63909]: pgmap v84: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 410 op/s 2024-09-16T09:43:03.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:03 smithi088 ceph-mon[52758]: pgmap v84: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 410 op/s 2024-09-16T09:43:03.892 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:03 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1[38201]: 2024-09-16T09:43:03.696+0000 7f165e530700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T09:43:03.892 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:03 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1[38201]: 2024-09-16T09:43:03.696+0000 7f165e530700 -1 osd.1 83 *** Got signal Terminated *** 2024-09-16T09:43:03.892 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:03 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1[38201]: 2024-09-16T09:43:03.696+0000 7f165e530700 -1 osd.1 83 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T09:43:04.564 INFO:teuthology.orchestra.run.smithi078.stdout: 5 173959 9.94 MB/sec execute 324 sec latency 74.473 ms 2024-09-16T09:43:04.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:04 smithi088 ceph-mon[52758]: osd.1 marked itself down and dead 2024-09-16T09:43:04.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:04 smithi078 ceph-mon[63909]: osd.1 marked itself down and dead 2024-09-16T09:43:05.434 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:05 smithi078 podman[70639]: 2024-09-16 09:43:05.132722895 +0000 UTC m=+1.585257485 container died d8b72a22a11b87b06443bc391cb0cd64bda3c88741d5a5b9f4de17aba58d25dd (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1, ceph=True, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., GIT_CLEAN=True, release=754, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, com.redhat.component=centos-stream-container, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-09-16T09:43:05.564 INFO:teuthology.orchestra.run.smithi078.stdout: 5 174014 9.92 MB/sec execute 325 sec latency 999.884 ms 2024-09-16T09:43:05.685 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:05 smithi078 ceph-mon[63909]: pgmap v85: 97 pgs: 97 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.4 MiB/s wr, 663 op/s 2024-09-16T09:43:05.685 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:05 smithi078 ceph-mon[63909]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T09:43:05.685 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:05 smithi078 ceph-mon[63909]: osdmap e84: 8 total, 7 up, 8 in 2024-09-16T09:43:05.685 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:05 smithi078 podman[70639]: 2024-09-16 09:43:05.557649549 +0000 UTC m=+2.010184141 container cleanup d8b72a22a11b87b06443bc391cb0cd64bda3c88741d5a5b9f4de17aba58d25dd (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1, GIT_BRANCH=HEAD, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, vcs-type=git, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, vendor=Red Hat, Inc., GIT_CLEAN=True, version=8, io.openshift.expose-services=) 2024-09-16T09:43:05.685 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:05 smithi078 bash[70639]: ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1 2024-09-16T09:43:05.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:05 smithi088 ceph-mon[52758]: pgmap v85: 97 pgs: 97 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.4 MiB/s wr, 663 op/s 2024-09-16T09:43:05.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:05 smithi088 ceph-mon[52758]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T09:43:05.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:05 smithi088 ceph-mon[52758]: osdmap e84: 8 total, 7 up, 8 in 2024-09-16T09:43:05.935 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:05 smithi078 podman[70657]: 2024-09-16 09:43:05.845060742 +0000 UTC m=+0.713400109 container remove d8b72a22a11b87b06443bc391cb0cd64bda3c88741d5a5b9f4de17aba58d25dd (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, release=754, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., ceph=True, vcs-type=git, version=8, RELEASE=HEAD, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-16T09:43:06.564 INFO:teuthology.orchestra.run.smithi078.stdout: 5 174526 9.94 MB/sec execute 326 sec latency 29.717 ms 2024-09-16T09:43:06.695 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:06 smithi078 ceph-mon[63909]: osdmap e85: 8 total, 7 up, 8 in 2024-09-16T09:43:06.696 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:06 smithi078 podman[70743]: 2024-09-16 09:43:06.390475381 +0000 UTC m=+0.025545968 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:43:06.696 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:06 smithi078 podman[70743]: 2024-09-16 09:43:06.579025252 +0000 UTC m=+0.214095831 container create bbd2ef83a5e8ef0b3ae17ffe4f37b7ef374e1c023c5868bc356b3535e52b94bf (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-16T09:43:06.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:06 smithi088 ceph-mon[52758]: osdmap e85: 8 total, 7 up, 8 in 2024-09-16T09:43:07.201 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:07 smithi078 podman[70743]: 2024-09-16 09:43:07.012632903 +0000 UTC m=+0.647703498 container init bbd2ef83a5e8ef0b3ae17ffe4f37b7ef374e1c023c5868bc356b3535e52b94bf (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-16T09:43:07.201 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:07 smithi078 podman[70743]: 2024-09-16 09:43:07.023589316 +0000 UTC m=+0.658659894 container start bbd2ef83a5e8ef0b3ae17ffe4f37b7ef374e1c023c5868bc356b3535e52b94bf (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-16T09:43:07.453 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:07 smithi078 podman[70743]: 2024-09-16 09:43:07.225478258 +0000 UTC m=+0.860548841 container attach bbd2ef83a5e8ef0b3ae17ffe4f37b7ef374e1c023c5868bc356b3535e52b94bf (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD) 2024-09-16T09:43:07.453 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:07 smithi078 conmon[70756]: conmon bbd2ef83a5e8ef0b3ae1 : Failed to open cgroups file: /sys/fs/cgroup/machine.slice/libpod-bbd2ef83a5e8ef0b3ae17ffe4f37b7ef374e1c023c5868bc356b3535e52b94bf.scope/container/memory.events 2024-09-16T09:43:07.564 INFO:teuthology.orchestra.run.smithi078.stdout: 5 175008 9.96 MB/sec execute 327 sec latency 30.637 ms 2024-09-16T09:43:07.707 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:07 smithi078 ceph-mon[63909]: pgmap v88: 97 pgs: 13 stale+active+clean, 84 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 4.7 MiB/s wr, 476 op/s 2024-09-16T09:43:07.708 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:07 smithi078 podman[70764]: 2024-09-16 09:43:07.460627514 +0000 UTC m=+0.027005560 container died bbd2ef83a5e8ef0b3ae17ffe4f37b7ef374e1c023c5868bc356b3535e52b94bf (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-16T09:43:07.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:07 smithi088 ceph-mon[52758]: pgmap v88: 97 pgs: 13 stale+active+clean, 84 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 4.7 MiB/s wr, 476 op/s 2024-09-16T09:43:08.564 INFO:teuthology.orchestra.run.smithi078.stdout: 5 175509 9.96 MB/sec execute 328 sec latency 39.746 ms 2024-09-16T09:43:08.720 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:08 smithi078 ceph-mon[63909]: Health check failed: Degraded data redundancy: 98/1701 objects degraded (5.761%), 15 pgs degraded (PG_DEGRADED) 2024-09-16T09:43:08.721 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:08 smithi078 podman[70764]: 2024-09-16 09:43:08.593859444 +0000 UTC m=+1.160237481 container remove bbd2ef83a5e8ef0b3ae17ffe4f37b7ef374e1c023c5868bc356b3535e52b94bf (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-16T09:43:08.721 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:08 smithi078 systemd[1]: ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.1.service: Deactivated successfully. 2024-09-16T09:43:08.721 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:08 smithi078 systemd[1]: Stopped Ceph osd.1 for a70c765e-740d-11ef-bceb-c7b262605968. 2024-09-16T09:43:08.721 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:08 smithi078 systemd[1]: ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.1.service: Consumed 2min 56.421s CPU time. 2024-09-16T09:43:08.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:08 smithi088 ceph-mon[52758]: Health check failed: Degraded data redundancy: 98/1701 objects degraded (5.761%), 15 pgs degraded (PG_DEGRADED) 2024-09-16T09:43:09.242 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:09 smithi078 systemd[1]: Starting Ceph osd.1 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:43:09.492 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:09 smithi078 podman[70857]: 2024-09-16 09:43:09.221401602 +0000 UTC m=+0.025321337 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:43:09.493 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:09 smithi078 podman[70857]: 2024-09-16 09:43:09.404000436 +0000 UTC m=+0.207920175 container create 7558b148ec5f786b4fc79d8ebf58f4dda933ea60618ecfd3a4656c041117a7d3 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1-activate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-16T09:43:09.564 INFO:teuthology.orchestra.run.smithi078.stdout: 5 175868 9.96 MB/sec execute 329 sec latency 44.571 ms 2024-09-16T09:43:09.744 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:09 smithi078 ceph-mon[63909]: pgmap v89: 97 pgs: 11 active+undersized, 5 stale+active+clean, 15 active+undersized+degraded, 66 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 5.3 MiB/s wr, 535 op/s; 98/1701 objects degraded (5.761%) 2024-09-16T09:43:09.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:09 smithi088 ceph-mon[52758]: pgmap v89: 97 pgs: 11 active+undersized, 5 stale+active+clean, 15 active+undersized+degraded, 66 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 5.3 MiB/s wr, 535 op/s; 98/1701 objects degraded (5.761%) 2024-09-16T09:43:10.250 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:10 smithi078 podman[70857]: 2024-09-16 09:43:10.179510384 +0000 UTC m=+0.983430131 container init 7558b148ec5f786b4fc79d8ebf58f4dda933ea60618ecfd3a4656c041117a7d3 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1-activate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6) 2024-09-16T09:43:10.250 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:10 smithi078 podman[70857]: 2024-09-16 09:43:10.191147394 +0000 UTC m=+0.995067134 container start 7558b148ec5f786b4fc79d8ebf58f4dda933ea60618ecfd3a4656c041117a7d3 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-16T09:43:10.503 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:10 smithi078 podman[70857]: 2024-09-16 09:43:10.334821526 +0000 UTC m=+1.138741261 container attach 7558b148ec5f786b4fc79d8ebf58f4dda933ea60618ecfd3a4656c041117a7d3 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1-activate, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6) 2024-09-16T09:43:10.504 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:10 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1-activate[70871]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:43:10.504 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:10 smithi078 bash[70857]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:43:10.504 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:10 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1-activate[70871]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:43:10.504 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:10 smithi078 bash[70857]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:43:10.564 INFO:teuthology.orchestra.run.smithi078.stdout: 5 176326 9.97 MB/sec execute 330 sec latency 43.045 ms 2024-09-16T09:43:11.564 INFO:teuthology.orchestra.run.smithi078.stdout: 5 176849 9.96 MB/sec execute 331 sec latency 46.314 ms 2024-09-16T09:43:11.775 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:11 smithi078 ceph-mon[63909]: pgmap v90: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.7 MiB/s wr, 755 op/s; 200/1725 objects degraded (11.594%) 2024-09-16T09:43:11.776 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:11 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1-activate[70871]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-16T09:43:11.776 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:11 smithi078 bash[70857]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-16T09:43:11.776 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:11 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1-activate[70871]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-16T09:43:11.776 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:11 smithi078 bash[70857]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-16T09:43:11.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:11 smithi088 ceph-mon[52758]: pgmap v90: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.7 MiB/s wr, 755 op/s; 200/1725 objects degraded (11.594%) 2024-09-16T09:43:12.278 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:12 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1-activate[70871]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-16T09:43:12.278 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:12 smithi078 bash[70857]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-16T09:43:12.278 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:12 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1-activate[70871]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-16T09:43:12.279 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:12 smithi078 bash[70857]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-16T09:43:12.279 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:12 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1-activate[70871]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-16T09:43:12.279 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:12 smithi078 bash[70857]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-16T09:43:12.279 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:12 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1-activate[70871]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-16T09:43:12.279 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:12 smithi078 bash[70857]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-16T09:43:12.279 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:12 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1-activate[70871]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-16T09:43:12.280 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:12 smithi078 bash[70857]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-16T09:43:12.280 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:12 smithi078 podman[71020]: 2024-09-16 09:43:12.14206755 +0000 UTC m=+0.026430523 container died 7558b148ec5f786b4fc79d8ebf58f4dda933ea60618ecfd3a4656c041117a7d3 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-16T09:43:12.564 INFO:teuthology.orchestra.run.smithi078.stdout: 5 177125 9.95 MB/sec execute 332 sec latency 44.469 ms 2024-09-16T09:43:13.034 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:13 smithi078 podman[71020]: 2024-09-16 09:43:13.009299988 +0000 UTC m=+0.893662949 container remove 7558b148ec5f786b4fc79d8ebf58f4dda933ea60618ecfd3a4656c041117a7d3 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-16T09:43:13.285 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:13 smithi078 podman[71090]: 2024-09-16 09:43:13.161618748 +0000 UTC m=+0.022594692 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:43:13.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:13 smithi078 ceph-mon[63909]: pgmap v91: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.1 MiB/s wr, 376 op/s; 200/1725 objects degraded (11.594%) 2024-09-16T09:43:13.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:13 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:43:13.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:13 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:43:13.564 INFO:teuthology.orchestra.run.smithi078.stdout: 5 177565 9.97 MB/sec execute 333 sec latency 57.542 ms 2024-09-16T09:43:13.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:13 smithi088 ceph-mon[52758]: pgmap v91: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.1 MiB/s wr, 376 op/s; 200/1725 objects degraded (11.594%) 2024-09-16T09:43:13.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:13 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:43:13.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:13 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:43:13.842 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:13 smithi078 podman[71090]: 2024-09-16 09:43:13.563408176 +0000 UTC m=+0.424384107 container create 12c137fcba709e979c75fe97d30e55655e30b8a73fbc369899cceccd1a5f90a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6) 2024-09-16T09:43:14.096 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:13 smithi078 podman[71090]: 2024-09-16 09:43:13.977446037 +0000 UTC m=+0.838421976 container init 12c137fcba709e979c75fe97d30e55655e30b8a73fbc369899cceccd1a5f90a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-16T09:43:14.096 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:13 smithi078 podman[71090]: 2024-09-16 09:43:13.986257488 +0000 UTC m=+0.847233425 container start 12c137fcba709e979c75fe97d30e55655e30b8a73fbc369899cceccd1a5f90a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_CLEAN=True) 2024-09-16T09:43:14.096 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:14 smithi078 ceph-osd[71108]: -- 172.21.15.78:0/1608581806 <== mon.0 v2:172.21.15.78:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b159ad81e0 con 0x55b158c73c00 2024-09-16T09:43:14.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:14 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 200/1725 objects degraded (11.594%), 24 pgs degraded (PG_DEGRADED) 2024-09-16T09:43:14.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:14 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:43:14.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:14 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:43:14.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:14 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:43:14.347 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:14 smithi078 bash[71090]: 12c137fcba709e979c75fe97d30e55655e30b8a73fbc369899cceccd1a5f90a0 2024-09-16T09:43:14.347 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:14 smithi078 systemd[1]: Started Ceph osd.1 for a70c765e-740d-11ef-bceb-c7b262605968. 2024-09-16T09:43:14.565 INFO:teuthology.orchestra.run.smithi078.stdout: 5 177996 9.97 MB/sec execute 334 sec latency 54.469 ms 2024-09-16T09:43:14.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:14 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 200/1725 objects degraded (11.594%), 24 pgs degraded (PG_DEGRADED) 2024-09-16T09:43:14.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:14 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:43:14.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:14 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:43:14.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:14 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:43:15.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:15 smithi078 ceph-mon[63909]: pgmap v92: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.4 MiB/s wr, 646 op/s; 192/1641 objects degraded (11.700%) 2024-09-16T09:43:15.564 INFO:teuthology.orchestra.run.smithi078.stdout: 5 178747 10.02 MB/sec execute 335 sec latency 30.946 ms 2024-09-16T09:43:15.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:15 smithi088 ceph-mon[52758]: pgmap v92: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.4 MiB/s wr, 646 op/s; 192/1641 objects degraded (11.700%) 2024-09-16T09:43:16.144 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:16 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1[71104]: 2024-09-16T09:43:16.129+0000 7f38dbb9e640 -1 Falling back to public interface 2024-09-16T09:43:16.564 INFO:teuthology.orchestra.run.smithi078.stdout: 5 179338 10.02 MB/sec execute 336 sec latency 29.599 ms 2024-09-16T09:43:16.842 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-16T09:43:17.427 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:17 smithi078 ceph-mon[63909]: pgmap v93: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.8 MiB/s wr, 584 op/s; 192/1641 objects degraded (11.700%) 2024-09-16T09:43:17.427 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:17 smithi078 ceph-mon[63909]: from='client.44185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:43:17.564 INFO:teuthology.orchestra.run.smithi078.stdout: 5 179562 10.01 MB/sec execute 337 sec latency 84.416 ms 2024-09-16T09:43:17.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:17 smithi088 ceph-mon[52758]: pgmap v93: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.8 MiB/s wr, 584 op/s; 192/1641 objects degraded (11.700%) 2024-09-16T09:43:17.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:17 smithi088 ceph-mon[52758]: from='client.44185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:43:17.965 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T09:43:17.966 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (4m) 43s ago 15m 17.6M - 0.27.0 11f11916f8cd 3ebb6832fbcc 2024-09-16T09:43:17.966 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (89s) 43s ago 15m 6899k - 19.3.0-4882-g66e33d6a 261f2dc7b20f 72965213139c 2024-09-16T09:43:17.966 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi088 smithi088 running (83s) 79s ago 13m 6891k - 19.3.0-4882-g66e33d6a 261f2dc7b20f eac0e6855b1b 2024-09-16T09:43:17.966 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (4m) 43s ago 13m 44.8M - dad864ee21e9 4a91b97e29b1 2024-09-16T09:43:17.966 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.hiotnn smithi078 running (8m) 43s ago 8m 37.4M - 17.2.0 e1d6a67b021e c43db7767ab9 2024-09-16T09:43:17.966 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi088.dbunob smithi088 running (8m) 79s ago 8m 17.0M - 17.2.0 e1d6a67b021e d235f1bfbb05 2024-09-16T09:43:17.966 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rrfxge smithi078 *:8443,9283,8765 running (3m) 43s ago 16m 511M - 19.3.0-4882-g66e33d6a 261f2dc7b20f 0d25f2f30f1e 2024-09-16T09:43:17.966 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi088.ncxfdz smithi088 *:8443,9283,8765 running (3m) 79s ago 13m 453M - 19.3.0-4882-g66e33d6a 261f2dc7b20f e2d03f701acd 2024-09-16T09:43:17.966 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (2m) 43s ago 17m 49.3M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f f5a3a0fec757 2024-09-16T09:43:17.966 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi088 smithi088 running (2m) 79s ago 13m 37.7M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f 2e552147327a 2024-09-16T09:43:17.966 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.wawosh smithi078 *:2049 running (7m) 43s ago 7m 280M - 4.0 e1d6a67b021e 2544ed9d67f5 2024-09-16T09:43:17.967 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi088.bnccmw smithi088 *:2049 running (7m) 79s ago 7m 81.6M - 4.0 e1d6a67b021e e606adb67049 2024-09-16T09:43:17.967 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (4m) 43s ago 13m 9.83M - 1.7.0 72c9c2088986 895ac3ccb2f3 2024-09-16T09:43:17.967 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi088 smithi088 *:9100 running (4m) 79s ago 13m 10.9M - 1.7.0 72c9c2088986 c84a2e50de53 2024-09-16T09:43:17.967 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (51s) 43s ago 12m 12.1M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 85008b499cb7 2024-09-16T09:43:17.967 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 starting - - - 1936M 2024-09-16T09:43:17.967 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (11m) 43s ago 11m 379M 1936M 17.2.0 e1d6a67b021e aa521853c9bf 2024-09-16T09:43:17.967 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (11m) 43s ago 11m 516M 1936M 17.2.0 e1d6a67b021e db62172fe90c 2024-09-16T09:43:17.967 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi088 running (10m) 79s ago 10m 505M 2704M 17.2.0 e1d6a67b021e 6f4e80fe127a 2024-09-16T09:43:17.967 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi088 running (10m) 79s ago 10m 322M 2704M 17.2.0 e1d6a67b021e 73e51edb7756 2024-09-16T09:43:17.967 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi088 running (10m) 79s ago 10m 503M 2704M 17.2.0 e1d6a67b021e 7c1459b1a2d9 2024-09-16T09:43:17.967 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi088 running (9m) 79s ago 9m 398M 2704M 17.2.0 e1d6a67b021e 37f619c2ddf9 2024-09-16T09:43:17.968 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (3m) 43s ago 13m 50.5M - 2.51.0 1d3b7f56885b 4c956ead9125 2024-09-16T09:43:18.443 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:18 smithi078 ceph-mon[63909]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:43:18.564 INFO:teuthology.orchestra.run.smithi078.stdout: 5 179960 10.02 MB/sec execute 338 sec latency 58.351 ms 2024-09-16T09:43:18.587 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:43:18.587 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-16T09:43:18.587 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:43:18.587 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:43:18.588 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-16T09:43:18.588 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:43:18.588 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:43:18.588 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-16T09:43:18.588 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-16T09:43:18.588 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 1 2024-09-16T09:43:18.588 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:43:18.588 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-16T09:43:18.588 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:43:18.588 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:43:18.588 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-16T09:43:18.588 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-09-16T09:43:18.589 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 5 2024-09-16T09:43:18.589 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-16T09:43:18.589 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:43:18.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:18 smithi088 ceph-mon[52758]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:43:19.129 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:43:19.129 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f", 2024-09-16T09:43:19.130 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-16T09:43:19.130 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T09:43:19.130 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-16T09:43:19.130 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-09-16T09:43:19.130 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-16T09:43:19.130 INFO:teuthology.orchestra.run.smithi078.stdout: "mon" 2024-09-16T09:43:19.130 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-16T09:43:19.130 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "7/23 daemons upgraded", 2024-09-16T09:43:19.130 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-09-16T09:43:19.130 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-16T09:43:19.130 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:43:19.465 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:19 smithi078 ceph-mon[63909]: from='client.34234 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:43:19.465 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:19 smithi078 ceph-mon[63909]: pgmap v94: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.8 MiB/s wr, 673 op/s; 182/1596 objects degraded (11.404%) 2024-09-16T09:43:19.466 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:19 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/1811350790' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:43:19.466 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:19 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 182/1596 objects degraded (11.404%), 24 pgs degraded (PG_DEGRADED) 2024-09-16T09:43:19.565 INFO:teuthology.orchestra.run.smithi078.stdout: 5 180471 10.02 MB/sec execute 339 sec latency 39.510 ms 2024-09-16T09:43:19.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:19 smithi088 ceph-mon[52758]: from='client.34234 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:43:19.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:19 smithi088 ceph-mon[52758]: pgmap v94: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.8 MiB/s wr, 673 op/s; 182/1596 objects degraded (11.404%) 2024-09-16T09:43:19.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:19 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/1811350790' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:43:19.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:19 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 182/1596 objects degraded (11.404%), 24 pgs degraded (PG_DEGRADED) 2024-09-16T09:43:19.867 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 182/1596 objects degraded (11.404%), 24 pgs degraded 2024-09-16T09:43:19.867 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-16T09:43:19.867 INFO:teuthology.orchestra.run.smithi078.stdout: osd.1 (root=default,host=smithi078) is down 2024-09-16T09:43:19.867 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 182/1596 objects degraded (11.404%), 24 pgs degraded 2024-09-16T09:43:19.867 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.0 is active+undersized+degraded, acting [7,0] 2024-09-16T09:43:19.867 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.2 is active+undersized+degraded, acting [5,6] 2024-09-16T09:43:19.867 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.4 is active+undersized+degraded, acting [0,2] 2024-09-16T09:43:19.867 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.6 is active+undersized+degraded, acting [6,4] 2024-09-16T09:43:19.867 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.8 is active+undersized+degraded, acting [7,0] 2024-09-16T09:43:19.867 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.9 is active+undersized+degraded, acting [7,3] 2024-09-16T09:43:19.868 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.a is active+undersized+degraded, acting [3,7] 2024-09-16T09:43:19.868 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.d is active+undersized+degraded, acting [4,3] 2024-09-16T09:43:19.868 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.10 is active+undersized+degraded, acting [2,0] 2024-09-16T09:43:19.868 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.11 is active+undersized+degraded, acting [6,4] 2024-09-16T09:43:19.868 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.15 is active+undersized+degraded, acting [5,0] 2024-09-16T09:43:19.868 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.0 is active+undersized+degraded, acting [2,6] 2024-09-16T09:43:19.868 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.4 is active+undersized+degraded, acting [2,5] 2024-09-16T09:43:19.868 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.6 is active+undersized+degraded, acting [0,4] 2024-09-16T09:43:19.868 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.8 is active+undersized+degraded, acting [3,7] 2024-09-16T09:43:19.868 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.a is active+undersized+degraded, acting [6,4] 2024-09-16T09:43:19.868 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.e is active+undersized+degraded, acting [7,4] 2024-09-16T09:43:19.868 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.16 is active+undersized+degraded, acting [5,7] 2024-09-16T09:43:19.868 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.18 is active+undersized+degraded, acting [3,0] 2024-09-16T09:43:19.869 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.19 is active+undersized+degraded, acting [3,4] 2024-09-16T09:43:19.869 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1a is active+undersized+degraded, acting [4,2] 2024-09-16T09:43:19.869 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1c is active+undersized+degraded, acting [5,4] 2024-09-16T09:43:19.869 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.a is active+undersized+degraded, acting [6,7] 2024-09-16T09:43:19.869 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.14 is active+undersized+degraded, acting [3,7] 2024-09-16T09:43:20.494 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:20 smithi078 ceph-mon[63909]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:43:20.495 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:20 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/1047880361' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:43:20.565 INFO:teuthology.orchestra.run.smithi078.stdout: 5 180846 10.01 MB/sec execute 340 sec latency 36.493 ms 2024-09-16T09:43:20.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:20 smithi088 ceph-mon[52758]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:43:20.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:20 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/1047880361' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:43:21.512 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:21 smithi078 ceph-mon[63909]: pgmap v95: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.1 MiB/s wr, 711 op/s; 176/1629 objects degraded (10.804%) 2024-09-16T09:43:21.565 INFO:teuthology.orchestra.run.smithi078.stdout: 5 181613 10.06 MB/sec execute 341 sec latency 22.654 ms 2024-09-16T09:43:21.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:21 smithi088 ceph-mon[52758]: pgmap v95: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.1 MiB/s wr, 711 op/s; 176/1629 objects degraded (10.804%) 2024-09-16T09:43:22.565 INFO:teuthology.orchestra.run.smithi078.stdout: 5 182058 10.08 MB/sec execute 342 sec latency 48.745 ms 2024-09-16T09:43:23.535 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:23 smithi078 ceph-mon[63909]: pgmap v96: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.6 MiB/s wr, 564 op/s; 176/1629 objects degraded (10.804%) 2024-09-16T09:43:23.565 INFO:teuthology.orchestra.run.smithi078.stdout: 5 182305 10.07 MB/sec execute 343 sec latency 50.449 ms 2024-09-16T09:43:23.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:23 smithi088 ceph-mon[52758]: pgmap v96: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.6 MiB/s wr, 564 op/s; 176/1629 objects degraded (10.804%) 2024-09-16T09:43:24.306 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:24 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1[71104]: 2024-09-16T09:43:24.156+0000 7f38dbb9e640 -1 osd.1 0 read_superblock omap replica is missing. 2024-09-16T09:43:24.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:24 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:43:24.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:24 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:43:24.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:24 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 176/1629 objects degraded (10.804%), 24 pgs degraded (PG_DEGRADED) 2024-09-16T09:43:24.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:24 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:43:24.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:24 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:43:24.565 INFO:teuthology.orchestra.run.smithi078.stdout: 5 182568 10.05 MB/sec execute 344 sec latency 75.029 ms 2024-09-16T09:43:24.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:24 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:43:24.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:24 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:43:24.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:24 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 176/1629 objects degraded (10.804%), 24 pgs degraded (PG_DEGRADED) 2024-09-16T09:43:24.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:24 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:43:24.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:24 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:43:25.565 INFO:teuthology.orchestra.run.smithi078.stdout: 5 182773 10.03 MB/sec execute 345 sec latency 47.755 ms 2024-09-16T09:43:25.567 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:25 smithi078 ceph-mon[63909]: pgmap v97: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 845 op/s; 187/1764 objects degraded (10.601%) 2024-09-16T09:43:25.567 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:25 smithi078 ceph-mon[63909]: from='osd.1 [v2:172.21.15.78:6810/999908339,v1:172.21.15.78:6811/999908339]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-16T09:43:25.568 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:25 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1[71104]: 2024-09-16T09:43:25.324+0000 7f38dbb9e640 -1 osd.1 83 log_to_monitors true 2024-09-16T09:43:25.568 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:43:25 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1[71104]: 2024-09-16T09:43:25.510+0000 7f38d3948640 -1 osd.1 83 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T09:43:25.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:25 smithi088 ceph-mon[52758]: pgmap v97: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 845 op/s; 187/1764 objects degraded (10.601%) 2024-09-16T09:43:25.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:25 smithi088 ceph-mon[52758]: from='osd.1 [v2:172.21.15.78:6810/999908339,v1:172.21.15.78:6811/999908339]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-16T09:43:26.565 INFO:teuthology.orchestra.run.smithi078.stdout: 5 183380 10.05 MB/sec execute 346 sec latency 69.436 ms 2024-09-16T09:43:26.584 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:26 smithi078 ceph-mon[63909]: from='osd.1 [v2:172.21.15.78:6810/999908339,v1:172.21.15.78:6811/999908339]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-16T09:43:26.584 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:26 smithi078 ceph-mon[63909]: osdmap e86: 8 total, 7 up, 8 in 2024-09-16T09:43:26.584 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:26 smithi078 ceph-mon[63909]: from='osd.1 [v2:172.21.15.78:6810/999908339,v1:172.21.15.78:6811/999908339]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]} : dispatch 2024-09-16T09:43:26.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:26 smithi088 ceph-mon[52758]: from='osd.1 [v2:172.21.15.78:6810/999908339,v1:172.21.15.78:6811/999908339]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-16T09:43:26.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:26 smithi088 ceph-mon[52758]: osdmap e86: 8 total, 7 up, 8 in 2024-09-16T09:43:26.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:26 smithi088 ceph-mon[52758]: from='osd.1 [v2:172.21.15.78:6810/999908339,v1:172.21.15.78:6811/999908339]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]} : dispatch 2024-09-16T09:43:27.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:27 smithi078 ceph-mon[63909]: pgmap v99: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.6 MiB/s wr, 702 op/s; 187/1764 objects degraded (10.601%) 2024-09-16T09:43:27.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:27 smithi078 ceph-mon[63909]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T09:43:27.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:27 smithi078 ceph-mon[63909]: osd.1 [v2:172.21.15.78:6810/999908339,v1:172.21.15.78:6811/999908339] boot 2024-09-16T09:43:27.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:27 smithi078 ceph-mon[63909]: osdmap e87: 8 total, 8 up, 8 in 2024-09-16T09:43:27.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:27 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-16T09:43:27.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:27 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:43:27.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:27 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:43:27.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:27 smithi078 ceph-mon[63909]: osdmap e88: 8 total, 8 up, 8 in 2024-09-16T09:43:27.565 INFO:teuthology.orchestra.run.smithi078.stdout: 5 183400 10.03 MB/sec execute 347 sec latency 1069.467 ms 2024-09-16T09:43:27.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:27 smithi088 ceph-mon[52758]: pgmap v99: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.6 MiB/s wr, 702 op/s; 187/1764 objects degraded (10.601%) 2024-09-16T09:43:27.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:27 smithi088 ceph-mon[52758]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T09:43:27.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:27 smithi088 ceph-mon[52758]: osd.1 [v2:172.21.15.78:6810/999908339,v1:172.21.15.78:6811/999908339] boot 2024-09-16T09:43:27.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:27 smithi088 ceph-mon[52758]: osdmap e87: 8 total, 8 up, 8 in 2024-09-16T09:43:27.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:27 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-16T09:43:27.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:27 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:43:27.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:27 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:43:27.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:27 smithi088 ceph-mon[52758]: osdmap e88: 8 total, 8 up, 8 in 2024-09-16T09:43:28.565 INFO:teuthology.orchestra.run.smithi078.stdout: 5 183405 10.00 MB/sec execute 348 sec latency 2043.597 ms 2024-09-16T09:43:29.559 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:29 smithi078 ceph-mon[63909]: pgmap v102: 97 pgs: 8 remapped+peering, 7 active+undersized, 12 active+undersized+degraded, 70 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 6.9 MiB/s wr, 620 op/s; 85/1662 objects degraded (5.114%); 0 B/s, 0 objects/s recovering 2024-09-16T09:43:29.559 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:29 smithi078 ceph-mon[63909]: osdmap e89: 8 total, 8 up, 8 in 2024-09-16T09:43:29.559 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:29 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 85/1662 objects degraded (5.114%), 12 pgs degraded (PG_DEGRADED) 2024-09-16T09:43:29.559 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:29 smithi078 ceph-mon[63909]: osdmap e90: 8 total, 8 up, 8 in 2024-09-16T09:43:29.565 INFO:teuthology.orchestra.run.smithi078.stdout: 5 183905 10.00 MB/sec execute 349 sec latency 105.464 ms 2024-09-16T09:43:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:29 smithi088 ceph-mon[52758]: pgmap v102: 97 pgs: 8 remapped+peering, 7 active+undersized, 12 active+undersized+degraded, 70 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 6.9 MiB/s wr, 620 op/s; 85/1662 objects degraded (5.114%); 0 B/s, 0 objects/s recovering 2024-09-16T09:43:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:29 smithi088 ceph-mon[52758]: osdmap e89: 8 total, 8 up, 8 in 2024-09-16T09:43:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:29 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 85/1662 objects degraded (5.114%), 12 pgs degraded (PG_DEGRADED) 2024-09-16T09:43:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:29 smithi088 ceph-mon[52758]: osdmap e90: 8 total, 8 up, 8 in 2024-09-16T09:43:30.565 INFO:teuthology.orchestra.run.smithi078.stdout: 5 184198 9.98 MB/sec execute 350 sec latency 63.975 ms 2024-09-16T09:43:31.108 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:31 smithi078 ceph-mon[63909]: pgmap v105: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 12 remapped+peering, 4 active+undersized, 4 active+undersized+degraded, 75 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 4.3 MiB/s wr, 495 op/s; 41/1545 objects degraded (2.654%); 1024 KiB/s, 0 objects/s recovering 2024-09-16T09:43:31.108 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:31 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:43:31.108 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:31 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:43:31.108 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:31 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:43:31.108 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:31 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:43:31.108 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:31 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:43:31.108 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:31 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:43:31.108 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:31 smithi078 ceph-mon[63909]: osdmap e91: 8 total, 8 up, 8 in 2024-09-16T09:43:31.108 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:31 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:43:31.109 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:31 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:43:31.109 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:31 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:43:31.109 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:31 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:43:31.109 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:31 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-16T09:43:31.109 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:31 smithi078 ceph-mon[63909]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-16T09:43:31.109 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:31 smithi078 ceph-mon[63909]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-09-16T09:43:31.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:31 smithi088 ceph-mon[52758]: pgmap v105: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 12 remapped+peering, 4 active+undersized, 4 active+undersized+degraded, 75 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 4.3 MiB/s wr, 495 op/s; 41/1545 objects degraded (2.654%); 1024 KiB/s, 0 objects/s recovering 2024-09-16T09:43:31.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:31 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:43:31.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:31 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:43:31.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:31 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:43:31.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:31 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:43:31.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:31 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:43:31.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:31 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:43:31.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:31 smithi088 ceph-mon[52758]: osdmap e91: 8 total, 8 up, 8 in 2024-09-16T09:43:31.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:31 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:43:31.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:31 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:43:31.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:31 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:43:31.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:31 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:43:31.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:31 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-16T09:43:31.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:31 smithi088 ceph-mon[52758]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-16T09:43:31.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:31 smithi088 ceph-mon[52758]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-09-16T09:43:31.565 INFO:teuthology.orchestra.run.smithi078.stdout: 5 184712 10.01 MB/sec execute 351 sec latency 72.907 ms 2024-09-16T09:43:32.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:32 smithi078 ceph-mon[63909]: osdmap e92: 8 total, 8 up, 8 in 2024-09-16T09:43:32.550 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:32 smithi088 ceph-mon[52758]: osdmap e92: 8 total, 8 up, 8 in 2024-09-16T09:43:32.565 INFO:teuthology.orchestra.run.smithi078.stdout: 5 185319 10.04 MB/sec execute 352 sec latency 364.085 ms 2024-09-16T09:43:33.374 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:33 smithi078 ceph-mon[63909]: pgmap v108: 97 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 10 remapped+peering, 85 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 3.1 MiB/s wr, 415 op/s; 30/1548 objects degraded (1.938%); 2.0 MiB/s, 60 keys/s, 3 objects/s recovering 2024-09-16T09:43:33.375 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:33 smithi078 ceph-mon[63909]: osdmap e93: 8 total, 8 up, 8 in 2024-09-16T09:43:33.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:33 smithi088 ceph-mon[52758]: pgmap v108: 97 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 10 remapped+peering, 85 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 3.1 MiB/s wr, 415 op/s; 30/1548 objects degraded (1.938%); 2.0 MiB/s, 60 keys/s, 3 objects/s recovering 2024-09-16T09:43:33.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:33 smithi088 ceph-mon[52758]: osdmap e93: 8 total, 8 up, 8 in 2024-09-16T09:43:33.565 INFO:teuthology.orchestra.run.smithi078.stdout: 5 185757 10.05 MB/sec execute 353 sec latency 994.629 ms 2024-09-16T09:43:34.387 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:34 smithi078 ceph-mon[63909]: osdmap e94: 8 total, 8 up, 8 in 2024-09-16T09:43:34.387 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:34 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 30/1548 objects degraded (1.938%), 1 pg degraded (PG_DEGRADED) 2024-09-16T09:43:34.387 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:34 smithi078 ceph-mon[63909]: osdmap e95: 8 total, 8 up, 8 in 2024-09-16T09:43:34.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:34 smithi088 ceph-mon[52758]: osdmap e94: 8 total, 8 up, 8 in 2024-09-16T09:43:34.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:34 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 30/1548 objects degraded (1.938%), 1 pg degraded (PG_DEGRADED) 2024-09-16T09:43:34.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:34 smithi088 ceph-mon[52758]: osdmap e95: 8 total, 8 up, 8 in 2024-09-16T09:43:34.565 INFO:teuthology.orchestra.run.smithi078.stdout: 5 186086 10.03 MB/sec execute 354 sec latency 880.187 ms 2024-09-16T09:43:35.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:35 smithi078 ceph-mon[63909]: pgmap v111: 97 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 peering, 5 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 7.9 MiB/s wr, 714 op/s; 146/1764 objects degraded (8.277%); 2.2 MiB/s, 60 keys/s, 40 objects/s recovering 2024-09-16T09:43:35.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:35 smithi078 ceph-mon[63909]: osdmap e96: 8 total, 8 up, 8 in 2024-09-16T09:43:35.550 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:35 smithi088 ceph-mon[52758]: pgmap v111: 97 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 peering, 5 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 7.9 MiB/s wr, 714 op/s; 146/1764 objects degraded (8.277%); 2.2 MiB/s, 60 keys/s, 40 objects/s recovering 2024-09-16T09:43:35.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:35 smithi088 ceph-mon[52758]: osdmap e96: 8 total, 8 up, 8 in 2024-09-16T09:43:35.565 INFO:teuthology.orchestra.run.smithi078.stdout: 5 186330 10.02 MB/sec execute 355 sec latency 376.972 ms 2024-09-16T09:43:36.565 INFO:teuthology.orchestra.run.smithi078.stdout: 5 186580 10.01 MB/sec execute 356 sec latency 995.489 ms 2024-09-16T09:43:37.180 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:37 smithi078 ceph-mon[63909]: pgmap v114: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 2 peering, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 8.1 MiB/s wr, 724 op/s; 146/1773 objects degraded (8.235%); 1.2 MiB/s, 44 objects/s recovering 2024-09-16T09:43:37.181 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:37 smithi078 ceph-mon[63909]: osdmap e97: 8 total, 8 up, 8 in 2024-09-16T09:43:37.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:37 smithi088 ceph-mon[52758]: pgmap v114: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 2 peering, 5 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 8.1 MiB/s wr, 724 op/s; 146/1773 objects degraded (8.235%); 1.2 MiB/s, 44 objects/s recovering 2024-09-16T09:43:37.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:37 smithi088 ceph-mon[52758]: osdmap e97: 8 total, 8 up, 8 in 2024-09-16T09:43:37.565 INFO:teuthology.orchestra.run.smithi078.stdout: 5 186809 10.00 MB/sec execute 357 sec latency 94.921 ms 2024-09-16T09:43:38.229 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:38 smithi078 ceph-mon[63909]: osdmap e98: 8 total, 8 up, 8 in 2024-09-16T09:43:38.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:38 smithi088 ceph-mon[52758]: osdmap e98: 8 total, 8 up, 8 in 2024-09-16T09:43:38.565 INFO:teuthology.orchestra.run.smithi078.stdout: 5 187103 9.99 MB/sec execute 358 sec latency 49.844 ms 2024-09-16T09:43:39.260 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:39 smithi078 ceph-mon[63909]: pgmap v117: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 2 peering, 2 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.6 MiB/s wr, 493 op/s; 71/1632 objects degraded (4.350%); 0 B/s, 29 objects/s recovering 2024-09-16T09:43:39.261 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:39 smithi078 ceph-mon[63909]: osdmap e99: 8 total, 8 up, 8 in 2024-09-16T09:43:39.261 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:39 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 71/1632 objects degraded (4.350%), 3 pgs degraded (PG_DEGRADED) 2024-09-16T09:43:39.261 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:39 smithi078 ceph-mon[63909]: osdmap e100: 8 total, 8 up, 8 in 2024-09-16T09:43:39.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:39 smithi088 ceph-mon[52758]: pgmap v117: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 2 peering, 2 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.6 MiB/s wr, 493 op/s; 71/1632 objects degraded (4.350%); 0 B/s, 29 objects/s recovering 2024-09-16T09:43:39.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:39 smithi088 ceph-mon[52758]: osdmap e99: 8 total, 8 up, 8 in 2024-09-16T09:43:39.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:39 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 71/1632 objects degraded (4.350%), 3 pgs degraded (PG_DEGRADED) 2024-09-16T09:43:39.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:39 smithi088 ceph-mon[52758]: osdmap e100: 8 total, 8 up, 8 in 2024-09-16T09:43:39.565 INFO:teuthology.orchestra.run.smithi078.stdout: 5 187446 9.98 MB/sec execute 359 sec latency 251.478 ms 2024-09-16T09:43:40.568 INFO:teuthology.orchestra.run.smithi078.stdout: 5 187746 9.97 MB/sec execute 360 sec latency 843.026 ms 2024-09-16T09:43:41.310 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:41 smithi078 ceph-mon[63909]: pgmap v120: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 4.8 MiB/s wr, 561 op/s; 38/1581 objects degraded (2.404%); 0 B/s, 29 objects/s recovering 2024-09-16T09:43:41.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:41 smithi078 ceph-mon[63909]: osdmap e101: 8 total, 8 up, 8 in 2024-09-16T09:43:41.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:41 smithi088 ceph-mon[52758]: pgmap v120: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 4.8 MiB/s wr, 561 op/s; 38/1581 objects degraded (2.404%); 0 B/s, 29 objects/s recovering 2024-09-16T09:43:41.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:41 smithi088 ceph-mon[52758]: osdmap e101: 8 total, 8 up, 8 in 2024-09-16T09:43:41.565 INFO:teuthology.orchestra.run.smithi078.stdout: 5 188041 9.96 MB/sec execute 361 sec latency 54.040 ms 2024-09-16T09:43:42.565 INFO:teuthology.orchestra.run.smithi078.stdout: 5 188312 9.97 MB/sec execute 362 sec latency 43.655 ms 2024-09-16T09:43:43.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:43 smithi078 ceph-mon[63909]: pgmap v122: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 95 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.3 MiB/s wr, 484 op/s; 5/1551 objects degraded (0.322%); 0 B/s, 31 objects/s recovering 2024-09-16T09:43:43.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:43 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:43:43.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:43 smithi088 ceph-mon[52758]: pgmap v122: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 95 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.3 MiB/s wr, 484 op/s; 5/1551 objects degraded (0.322%); 0 B/s, 31 objects/s recovering 2024-09-16T09:43:43.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:43 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:43:43.565 INFO:teuthology.orchestra.run.smithi078.stdout: 5 188780 9.99 MB/sec execute 363 sec latency 50.352 ms 2024-09-16T09:43:44.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:44 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 5/1551 objects degraded (0.322%), 1 pg degraded (PG_DEGRADED) 2024-09-16T09:43:44.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:44 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 5/1551 objects degraded (0.322%), 1 pg degraded (PG_DEGRADED) 2024-09-16T09:43:44.565 INFO:teuthology.orchestra.run.smithi078.stdout: 5 189252 10.01 MB/sec execute 364 sec latency 38.428 ms 2024-09-16T09:43:45.368 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:45 smithi078 ceph-mon[63909]: pgmap v123: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 4.8 MiB/s wr, 432 op/s; 0 B/s, 13 objects/s recovering 2024-09-16T09:43:45.369 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:45 smithi078 ceph-mon[63909]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/1551 objects degraded (0.322%), 1 pg degraded) 2024-09-16T09:43:45.369 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:45 smithi078 ceph-mon[63909]: Cluster is now healthy 2024-09-16T09:43:45.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:45 smithi088 ceph-mon[52758]: pgmap v123: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 13 KiB/s rd, 4.8 MiB/s wr, 432 op/s; 0 B/s, 13 objects/s recovering 2024-09-16T09:43:45.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:45 smithi088 ceph-mon[52758]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/1551 objects degraded (0.322%), 1 pg degraded) 2024-09-16T09:43:45.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:45 smithi088 ceph-mon[52758]: Cluster is now healthy 2024-09-16T09:43:45.565 INFO:teuthology.orchestra.run.smithi078.stdout: 5 189601 9.99 MB/sec execute 365 sec latency 33.276 ms 2024-09-16T09:43:46.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:46 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-16T09:43:46.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:46 smithi078 ceph-mon[63909]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-16T09:43:46.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:46 smithi078 ceph-mon[63909]: Upgrade: osd.2 is safe to restart 2024-09-16T09:43:46.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:46 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-16T09:43:46.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:46 smithi088 ceph-mon[52758]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-16T09:43:46.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:46 smithi088 ceph-mon[52758]: Upgrade: osd.2 is safe to restart 2024-09-16T09:43:46.565 INFO:teuthology.orchestra.run.smithi078.stdout: 5 189831 9.97 MB/sec execute 366 sec latency 64.853 ms 2024-09-16T09:43:47.390 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:47 smithi078 ceph-mon[63909]: pgmap v124: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.5 MiB/s wr, 318 op/s; 0 B/s, 6 objects/s recovering 2024-09-16T09:43:47.550 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:47 smithi088 ceph-mon[52758]: pgmap v124: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.5 MiB/s wr, 318 op/s; 0 B/s, 6 objects/s recovering 2024-09-16T09:43:47.565 INFO:teuthology.orchestra.run.smithi078.stdout: 5 190096 9.97 MB/sec execute 367 sec latency 56.596 ms 2024-09-16T09:43:48.566 INFO:teuthology.orchestra.run.smithi078.stdout: 5 190412 9.97 MB/sec execute 368 sec latency 53.490 ms 2024-09-16T09:43:49.229 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:48 smithi078 ceph-mon[63909]: pgmap v125: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.9 MiB/s wr, 314 op/s; 0 B/s, 5 objects/s recovering 2024-09-16T09:43:49.229 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:48 smithi078 ceph-mon[63909]: Upgrade: Updating osd.2 2024-09-16T09:43:49.229 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:48 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:43:49.229 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:48 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-16T09:43:49.229 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:48 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:43:49.229 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:48 smithi078 ceph-mon[63909]: Deploying daemon osd.2 on smithi078 2024-09-16T09:43:49.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:48 smithi088 ceph-mon[52758]: pgmap v125: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.9 MiB/s wr, 314 op/s; 0 B/s, 5 objects/s recovering 2024-09-16T09:43:49.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:48 smithi088 ceph-mon[52758]: Upgrade: Updating osd.2 2024-09-16T09:43:49.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:48 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:43:49.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:48 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-16T09:43:49.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:48 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:43:49.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:48 smithi088 ceph-mon[52758]: Deploying daemon osd.2 on smithi078 2024-09-16T09:43:49.566 INFO:teuthology.orchestra.run.smithi078.stdout: 5 190781 9.96 MB/sec execute 369 sec latency 55.077 ms 2024-09-16T09:43:50.428 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-16T09:43:50.566 INFO:teuthology.orchestra.run.smithi078.stdout: 5 191223 9.96 MB/sec execute 370 sec latency 56.269 ms 2024-09-16T09:43:51.006 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:50 smithi078 ceph-mon[63909]: pgmap v126: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.0 MiB/s wr, 490 op/s; 0 B/s, 4 objects/s recovering 2024-09-16T09:43:51.007 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:50 smithi078 ceph-mon[63909]: from='client.34252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:43:51.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:50 smithi088 ceph-mon[52758]: pgmap v126: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.0 MiB/s wr, 490 op/s; 0 B/s, 4 objects/s recovering 2024-09-16T09:43:51.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:50 smithi088 ceph-mon[52758]: from='client.34252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:43:51.516 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T09:43:51.516 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (5m) 28s ago 16m 15.9M - 0.27.0 11f11916f8cd 3ebb6832fbcc 2024-09-16T09:43:51.516 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (2m) 28s ago 16m 6899k - 19.3.0-4882-g66e33d6a 261f2dc7b20f 72965213139c 2024-09-16T09:43:51.516 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi088 smithi088 running (117s) 112s ago 14m 6891k - 19.3.0-4882-g66e33d6a 261f2dc7b20f eac0e6855b1b 2024-09-16T09:43:51.517 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (5m) 28s ago 14m 44.8M - dad864ee21e9 4a91b97e29b1 2024-09-16T09:43:51.517 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.hiotnn smithi078 running (8m) 28s ago 8m 37.6M - 17.2.0 e1d6a67b021e c43db7767ab9 2024-09-16T09:43:51.517 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi088.dbunob smithi088 running (8m) 112s ago 8m 17.0M - 17.2.0 e1d6a67b021e d235f1bfbb05 2024-09-16T09:43:51.517 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rrfxge smithi078 *:8443,9283,8765 running (4m) 28s ago 17m 516M - 19.3.0-4882-g66e33d6a 261f2dc7b20f 0d25f2f30f1e 2024-09-16T09:43:51.517 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi088.ncxfdz smithi088 *:8443,9283,8765 running (3m) 112s ago 14m 453M - 19.3.0-4882-g66e33d6a 261f2dc7b20f e2d03f701acd 2024-09-16T09:43:51.517 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (3m) 28s ago 17m 51.9M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f f5a3a0fec757 2024-09-16T09:43:51.517 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi088 smithi088 running (2m) 112s ago 14m 37.7M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f 2e552147327a 2024-09-16T09:43:51.517 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.wawosh smithi078 *:2049 running (8m) 28s ago 8m 278M - 4.0 e1d6a67b021e 2544ed9d67f5 2024-09-16T09:43:51.517 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi088.bnccmw smithi088 *:2049 running (8m) 112s ago 8m 81.6M - 4.0 e1d6a67b021e e606adb67049 2024-09-16T09:43:51.517 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (5m) 28s ago 14m 10.3M - 1.7.0 72c9c2088986 895ac3ccb2f3 2024-09-16T09:43:51.517 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi088 smithi088 *:9100 running (4m) 112s ago 13m 10.9M - 1.7.0 72c9c2088986 c84a2e50de53 2024-09-16T09:43:51.517 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (85s) 28s ago 13m 242M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 85008b499cb7 2024-09-16T09:43:51.517 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (38s) 28s ago 12m 11.6M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 12c137fcba70 2024-09-16T09:43:51.517 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (12m) 28s ago 12m 438M 1936M 17.2.0 e1d6a67b021e aa521853c9bf 2024-09-16T09:43:51.518 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (11m) 28s ago 11m 566M 1936M 17.2.0 e1d6a67b021e db62172fe90c 2024-09-16T09:43:51.518 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi088 running (11m) 112s ago 11m 505M 2704M 17.2.0 e1d6a67b021e 6f4e80fe127a 2024-09-16T09:43:51.518 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi088 running (11m) 112s ago 11m 322M 2704M 17.2.0 e1d6a67b021e 73e51edb7756 2024-09-16T09:43:51.518 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi088 running (10m) 112s ago 10m 503M 2704M 17.2.0 e1d6a67b021e 7c1459b1a2d9 2024-09-16T09:43:51.518 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi088 running (10m) 112s ago 10m 398M 2704M 17.2.0 e1d6a67b021e 37f619c2ddf9 2024-09-16T09:43:51.518 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (3m) 28s ago 13m 50.8M - 2.51.0 1d3b7f56885b 4c956ead9125 2024-09-16T09:43:51.566 INFO:teuthology.orchestra.run.smithi078.stdout: 5 191640 9.96 MB/sec execute 371 sec latency 43.373 ms 2024-09-16T09:43:52.012 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:51 smithi078 ceph-mon[63909]: from='client.34256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:43:52.013 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:51 smithi078 ceph-mon[63909]: from='client.34260 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:43:52.280 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:43:52.280 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-16T09:43:52.280 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:43:52.280 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:43:52.280 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-16T09:43:52.280 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:43:52.280 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:43:52.280 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-16T09:43:52.280 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-16T09:43:52.280 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:43:52.281 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:43:52.281 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-16T09:43:52.281 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:43:52.281 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:43:52.281 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-16T09:43:52.281 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-09-16T09:43:52.281 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 6 2024-09-16T09:43:52.281 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-16T09:43:52.281 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:43:52.300 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:51 smithi088 ceph-mon[52758]: from='client.34256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:43:52.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:51 smithi088 ceph-mon[52758]: from='client.34260 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:43:52.566 INFO:teuthology.orchestra.run.smithi078.stdout: 5 192198 9.99 MB/sec execute 372 sec latency 29.288 ms 2024-09-16T09:43:52.935 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:43:52.935 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f", 2024-09-16T09:43:52.936 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-16T09:43:52.936 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T09:43:52.936 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-16T09:43:52.936 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-09-16T09:43:52.936 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-16T09:43:52.936 INFO:teuthology.orchestra.run.smithi078.stdout: "mon" 2024-09-16T09:43:52.936 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-16T09:43:52.936 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "8/23 daemons upgraded", 2024-09-16T09:43:52.936 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-09-16T09:43:52.936 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-16T09:43:52.936 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:43:53.031 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:43:52 smithi078 systemd[1]: Stopping Ceph osd.2 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:43:53.032 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:52 smithi078 ceph-mon[63909]: pgmap v127: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 421 op/s; 0 B/s, 4 objects/s recovering 2024-09-16T09:43:53.032 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:52 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/2827603916' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:43:53.032 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:52 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:43:53.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:52 smithi088 ceph-mon[52758]: pgmap v127: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 421 op/s; 0 B/s, 4 objects/s recovering 2024-09-16T09:43:53.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:52 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/2827603916' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:43:53.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:52 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:43:53.360 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:43:53 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2[41876]: 2024-09-16T09:43:53.300+0000 7f6a7836b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T09:43:53.360 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:43:53 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2[41876]: 2024-09-16T09:43:53.300+0000 7f6a7836b700 -1 osd.2 101 *** Got signal Terminated *** 2024-09-16T09:43:53.360 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:43:53 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2[41876]: 2024-09-16T09:43:53.300+0000 7f6a7836b700 -1 osd.2 101 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T09:43:53.566 INFO:teuthology.orchestra.run.smithi078.stdout: 5 192683 10.01 MB/sec execute 373 sec latency 31.897 ms 2024-09-16T09:43:53.659 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-16T09:43:54.041 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:53 smithi078 ceph-mon[63909]: from='client.34268 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:43:54.041 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:53 smithi078 ceph-mon[63909]: osd.2 marked itself down and dead 2024-09-16T09:43:54.042 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:53 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/3780470412' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:43:54.300 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:53 smithi088 ceph-mon[52758]: from='client.34268 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:43:54.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:53 smithi088 ceph-mon[52758]: osd.2 marked itself down and dead 2024-09-16T09:43:54.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:53 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/3780470412' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:43:54.566 INFO:teuthology.orchestra.run.smithi078.stdout: 5 193130 10.00 MB/sec execute 374 sec latency 557.417 ms 2024-09-16T09:43:54.811 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:43:54 smithi078 podman[73079]: 2024-09-16 09:43:54.69482831 +0000 UTC m=+1.588395526 container died aa521853c9bfce5ae5c67db8295b3ee687768e4453310569efe6fbe56097607f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , release=754, architecture=x86_64, vcs-type=git, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, version=8, RELEASE=HEAD) 2024-09-16T09:43:55.236 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:54 smithi078 ceph-mon[63909]: pgmap v128: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.5 MiB/s wr, 615 op/s; 0 B/s, 1 objects/s recovering 2024-09-16T09:43:55.237 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:54 smithi078 ceph-mon[63909]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T09:43:55.237 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:54 smithi078 ceph-mon[63909]: osdmap e102: 8 total, 7 up, 8 in 2024-09-16T09:43:55.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:54 smithi088 ceph-mon[52758]: pgmap v128: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.5 MiB/s wr, 615 op/s; 0 B/s, 1 objects/s recovering 2024-09-16T09:43:55.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:54 smithi088 ceph-mon[52758]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T09:43:55.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:54 smithi088 ceph-mon[52758]: osdmap e102: 8 total, 7 up, 8 in 2024-09-16T09:43:55.488 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:43:55 smithi078 podman[73079]: 2024-09-16 09:43:55.462874158 +0000 UTC m=+2.356441380 container cleanup aa521853c9bfce5ae5c67db8295b3ee687768e4453310569efe6fbe56097607f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, release=754, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, vcs-type=git, GIT_CLEAN=True, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0) 2024-09-16T09:43:55.488 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:43:55 smithi078 bash[73079]: ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2 2024-09-16T09:43:55.566 INFO:teuthology.orchestra.run.smithi078.stdout: 5 193359 9.99 MB/sec execute 375 sec latency 1007.057 ms 2024-09-16T09:43:55.997 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:55 smithi078 ceph-mon[63909]: osdmap e103: 8 total, 7 up, 8 in 2024-09-16T09:43:55.998 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:43:55 smithi078 podman[73119]: 2024-09-16 09:43:55.857966064 +0000 UTC m=+1.156616344 container remove aa521853c9bfce5ae5c67db8295b3ee687768e4453310569efe6fbe56097607f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2, ceph=True, vcs-type=git, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , name=centos-stream, architecture=x86_64, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, release=754, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc.) 2024-09-16T09:43:56.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:55 smithi088 ceph-mon[52758]: osdmap e103: 8 total, 7 up, 8 in 2024-09-16T09:43:56.505 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:43:56 smithi078 podman[73202]: 2024-09-16 09:43:56.365558653 +0000 UTC m=+0.025306500 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:43:56.566 INFO:teuthology.orchestra.run.smithi078.stdout: 5 193668 9.99 MB/sec execute 376 sec latency 34.536 ms 2024-09-16T09:43:56.779 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:43:56 smithi078 podman[73202]: 2024-09-16 09:43:56.554287775 +0000 UTC m=+0.214035613 container create 27a3fe523d8bbcd469c6f2d4ecf3568b54c8d78a2b5658857b66aab3a84c452c (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, CEPH_POINT_RELEASE=, ceph=True) 2024-09-16T09:43:57.044 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:56 smithi078 ceph-mon[63909]: pgmap v131: 97 pgs: 2 active+undersized, 3 stale+active+clean, 4 active+undersized+degraded, 88 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.5 MiB/s wr, 729 op/s; 40/1743 objects degraded (2.295%) 2024-09-16T09:43:57.044 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:56 smithi078 ceph-mon[63909]: Health check failed: Degraded data redundancy: 40/1743 objects degraded (2.295%), 4 pgs degraded (PG_DEGRADED) 2024-09-16T09:43:57.044 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:43:56 smithi078 podman[73202]: 2024-09-16 09:43:56.922054589 +0000 UTC m=+0.581802438 container init 27a3fe523d8bbcd469c6f2d4ecf3568b54c8d78a2b5658857b66aab3a84c452c (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6) 2024-09-16T09:43:57.044 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:43:56 smithi078 podman[73202]: 2024-09-16 09:43:56.934724031 +0000 UTC m=+0.594471870 container start 27a3fe523d8bbcd469c6f2d4ecf3568b54c8d78a2b5658857b66aab3a84c452c (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2-deactivate, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-16T09:43:57.044 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:43:57 smithi078 podman[73202]: 2024-09-16 09:43:57.022050574 +0000 UTC m=+0.681798411 container attach 27a3fe523d8bbcd469c6f2d4ecf3568b54c8d78a2b5658857b66aab3a84c452c (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-16T09:43:57.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:56 smithi088 ceph-mon[52758]: pgmap v131: 97 pgs: 2 active+undersized, 3 stale+active+clean, 4 active+undersized+degraded, 88 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.5 MiB/s wr, 729 op/s; 40/1743 objects degraded (2.295%) 2024-09-16T09:43:57.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:56 smithi088 ceph-mon[52758]: Health check failed: Degraded data redundancy: 40/1743 objects degraded (2.295%), 4 pgs degraded (PG_DEGRADED) 2024-09-16T09:43:57.310 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:43:57 smithi078 podman[73202]: 2024-09-16 09:43:57.27980371 +0000 UTC m=+0.939551541 container died 27a3fe523d8bbcd469c6f2d4ecf3568b54c8d78a2b5658857b66aab3a84c452c (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-16T09:43:57.566 INFO:teuthology.orchestra.run.smithi078.stdout: 5 194026 9.99 MB/sec execute 377 sec latency 72.696 ms 2024-09-16T09:43:58.349 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:58 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:43:58.349 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:58 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:43:58.566 INFO:teuthology.orchestra.run.smithi078.stdout: 5 194448 9.98 MB/sec execute 378 sec latency 45.578 ms 2024-09-16T09:43:58.604 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:43:58 smithi078 podman[73223]: 2024-09-16 09:43:58.434160065 +0000 UTC m=+1.141624869 container remove 27a3fe523d8bbcd469c6f2d4ecf3568b54c8d78a2b5658857b66aab3a84c452c (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-16T09:43:58.604 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:43:58 smithi078 systemd[1]: ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.2.service: Deactivated successfully. 2024-09-16T09:43:58.604 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:43:58 smithi078 systemd[1]: Stopped Ceph osd.2 for a70c765e-740d-11ef-bceb-c7b262605968. 2024-09-16T09:43:58.604 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:43:58 smithi078 systemd[1]: ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.2.service: Consumed 2min 26.609s CPU time. 2024-09-16T09:43:58.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:58 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:43:58.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:58 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:43:58.855 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:43:58 smithi078 systemd[1]: Starting Ceph osd.2 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:43:59.121 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:43:59 smithi078 podman[73320]: 2024-09-16 09:43:58.928083205 +0000 UTC m=+0.020684855 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:43:59.122 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:43:59 smithi078 podman[73320]: 2024-09-16 09:43:59.057277228 +0000 UTC m=+0.149878865 container create 30c5eef49c48c0adc000fb3101a09b8411d8f62efcc70c8b2abccde9714b87d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-16T09:43:59.380 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:43:59 smithi078 ceph-mon[63909]: pgmap v132: 97 pgs: 5 active+undersized, 16 active+undersized+degraded, 76 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.6 MiB/s wr, 525 op/s; 121/1695 objects degraded (7.139%) 2024-09-16T09:43:59.566 INFO:teuthology.orchestra.run.smithi078.stdout: 5 194687 9.97 MB/sec execute 379 sec latency 68.910 ms 2024-09-16T09:43:59.631 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:43:59 smithi078 podman[73320]: 2024-09-16 09:43:59.404752933 +0000 UTC m=+0.497354583 container init 30c5eef49c48c0adc000fb3101a09b8411d8f62efcc70c8b2abccde9714b87d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2-activate, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_CLEAN=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-16T09:43:59.631 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:43:59 smithi078 podman[73320]: 2024-09-16 09:43:59.411377331 +0000 UTC m=+0.503978968 container start 30c5eef49c48c0adc000fb3101a09b8411d8f62efcc70c8b2abccde9714b87d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.build-date=20240731, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-16T09:43:59.631 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:43:59 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2-activate[73335]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:43:59.631 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:43:59 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2-activate[73335]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:43:59.631 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:43:59 smithi078 bash[73320]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:43:59.631 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:43:59 smithi078 bash[73320]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:43:59.631 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:43:59 smithi078 podman[73320]: 2024-09-16 09:43:59.584295185 +0000 UTC m=+0.676896822 container attach 30c5eef49c48c0adc000fb3101a09b8411d8f62efcc70c8b2abccde9714b87d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-16T09:43:59.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:43:59 smithi088 ceph-mon[52758]: pgmap v132: 97 pgs: 5 active+undersized, 16 active+undersized+degraded, 76 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.6 MiB/s wr, 525 op/s; 121/1695 objects degraded (7.139%) 2024-09-16T09:44:00.566 INFO:teuthology.orchestra.run.smithi078.stdout: 5 194935 9.97 MB/sec execute 380 sec latency 58.556 ms 2024-09-16T09:44:00.663 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:44:00 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2-activate[73335]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-16T09:44:00.664 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:44:00 smithi078 bash[73320]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-16T09:44:00.664 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:44:00 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2-activate[73335]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-16T09:44:00.664 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:44:00 smithi078 bash[73320]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-16T09:44:00.917 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:44:00 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2-activate[73335]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-16T09:44:00.917 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:44:00 smithi078 bash[73320]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-16T09:44:00.917 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:44:00 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2-activate[73335]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-16T09:44:00.917 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:44:00 smithi078 bash[73320]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-16T09:44:00.917 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:44:00 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2-activate[73335]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-16T09:44:00.918 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:44:00 smithi078 bash[73320]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-16T09:44:00.918 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:44:00 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2-activate[73335]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-16T09:44:00.918 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:44:00 smithi078 bash[73320]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-16T09:44:00.918 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:44:00 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2-activate[73335]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-16T09:44:00.918 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:44:00 smithi078 bash[73320]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-16T09:44:00.918 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:44:00 smithi078 podman[73320]: 2024-09-16 09:44:00.855573899 +0000 UTC m=+1.948175532 container died 30c5eef49c48c0adc000fb3101a09b8411d8f62efcc70c8b2abccde9714b87d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-16T09:44:01.418 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:01 smithi078 ceph-mon[63909]: pgmap v133: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.8 MiB/s wr, 713 op/s; 157/1641 objects degraded (9.567%) 2024-09-16T09:44:01.566 INFO:teuthology.orchestra.run.smithi078.stdout: 5 195342 9.97 MB/sec execute 381 sec latency 63.755 ms 2024-09-16T09:44:01.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:01 smithi088 ceph-mon[52758]: pgmap v133: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.8 MiB/s wr, 713 op/s; 157/1641 objects degraded (9.567%) 2024-09-16T09:44:02.445 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:02 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 154/1620 objects degraded (9.506%), 19 pgs degraded (PG_DEGRADED) 2024-09-16T09:44:02.566 INFO:teuthology.orchestra.run.smithi078.stdout: 5 195755 9.98 MB/sec execute 382 sec latency 35.024 ms 2024-09-16T09:44:02.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:02 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 154/1620 objects degraded (9.506%), 19 pgs degraded (PG_DEGRADED) 2024-09-16T09:44:02.956 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:44:02 smithi078 podman[73478]: 2024-09-16 09:44:02.907167892 +0000 UTC m=+2.045747247 container remove 30c5eef49c48c0adc000fb3101a09b8411d8f62efcc70c8b2abccde9714b87d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, io.buildah.version=1.37.2) 2024-09-16T09:44:03.208 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:44:03 smithi078 podman[73529]: 2024-09-16 09:44:03.070832604 +0000 UTC m=+0.026166356 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:44:03.459 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:03 smithi078 ceph-mon[63909]: pgmap v134: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.5 MiB/s wr, 412 op/s; 154/1620 objects degraded (9.506%) 2024-09-16T09:44:03.459 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:44:03 smithi078 podman[73529]: 2024-09-16 09:44:03.289142854 +0000 UTC m=+0.244476588 container create 23a8b69c5c64a31a2d1ccd324b5bada8c6d962d9e11d5ca6fd78e325e3911275 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-16T09:44:03.566 INFO:teuthology.orchestra.run.smithi078.stdout: 5 196077 9.99 MB/sec execute 383 sec latency 50.261 ms 2024-09-16T09:44:03.714 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:44:03 smithi078 podman[73529]: 2024-09-16 09:44:03.692111983 +0000 UTC m=+0.647445720 container init 23a8b69c5c64a31a2d1ccd324b5bada8c6d962d9e11d5ca6fd78e325e3911275 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-16T09:44:03.714 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:44:03 smithi078 podman[73529]: 2024-09-16 09:44:03.69756159 +0000 UTC m=+0.652895323 container start 23a8b69c5c64a31a2d1ccd324b5bada8c6d962d9e11d5ca6fd78e325e3911275 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2) 2024-09-16T09:44:03.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:03 smithi088 ceph-mon[52758]: pgmap v134: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.5 MiB/s wr, 412 op/s; 154/1620 objects degraded (9.506%) 2024-09-16T09:44:03.972 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:44:03 smithi078 ceph-osd[73561]: -- 172.21.15.78:0/3524090088 <== mon.0 v2:172.21.15.78:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55613bcf0000 con 0x55613ae81c00 2024-09-16T09:44:03.972 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:44:03 smithi078 bash[73529]: 23a8b69c5c64a31a2d1ccd324b5bada8c6d962d9e11d5ca6fd78e325e3911275 2024-09-16T09:44:03.972 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:44:03 smithi078 systemd[1]: Started Ceph osd.2 for a70c765e-740d-11ef-bceb-c7b262605968. 2024-09-16T09:44:04.566 INFO:teuthology.orchestra.run.smithi078.stdout: 5 196374 9.98 MB/sec execute 384 sec latency 36.932 ms 2024-09-16T09:44:04.990 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:04 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:04.991 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:04 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:04.991 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:04 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:44:04.991 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:04 smithi078 ceph-mon[63909]: pgmap v135: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.4 MiB/s wr, 561 op/s; 148/1689 objects degraded (8.763%) 2024-09-16T09:44:05.300 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:04 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:05.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:04 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:05.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:04 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:44:05.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:04 smithi088 ceph-mon[52758]: pgmap v135: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.4 MiB/s wr, 561 op/s; 148/1689 objects degraded (8.763%) 2024-09-16T09:44:05.566 INFO:teuthology.orchestra.run.smithi078.stdout: 5 196699 9.97 MB/sec execute 385 sec latency 95.483 ms 2024-09-16T09:44:06.295 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:44:06 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2[73557]: 2024-09-16T09:44:06.065+0000 7f745af44640 -1 Falling back to public interface 2024-09-16T09:44:06.566 INFO:teuthology.orchestra.run.smithi078.stdout: 5 196922 9.95 MB/sec execute 386 sec latency 39.575 ms 2024-09-16T09:44:07.073 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:07 smithi078 ceph-mon[63909]: pgmap v136: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.1 MiB/s wr, 519 op/s; 150/1707 objects degraded (8.787%) 2024-09-16T09:44:07.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:07 smithi088 ceph-mon[52758]: pgmap v136: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.1 MiB/s wr, 519 op/s; 150/1707 objects degraded (8.787%) 2024-09-16T09:44:07.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 197111 9.94 MB/sec execute 387 sec latency 59.758 ms 2024-09-16T09:44:08.351 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:08 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:08.566 INFO:teuthology.orchestra.run.smithi078.stdout: 5 197807 9.96 MB/sec execute 388 sec latency 42.930 ms 2024-09-16T09:44:08.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:08 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:09.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:09 smithi078 ceph-mon[63909]: pgmap v137: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.0 MiB/s wr, 496 op/s; 142/1581 objects degraded (8.982%) 2024-09-16T09:44:09.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:09 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 142/1581 objects degraded (8.982%), 19 pgs degraded (PG_DEGRADED) 2024-09-16T09:44:09.566 INFO:teuthology.orchestra.run.smithi078.stdout: 5 198469 9.97 MB/sec execute 389 sec latency 25.412 ms 2024-09-16T09:44:09.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:09 smithi088 ceph-mon[52758]: pgmap v137: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.0 MiB/s wr, 496 op/s; 142/1581 objects degraded (8.982%) 2024-09-16T09:44:09.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:09 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 142/1581 objects degraded (8.982%), 19 pgs degraded (PG_DEGRADED) 2024-09-16T09:44:10.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 199330 10.02 MB/sec execute 390 sec latency 28.978 ms 2024-09-16T09:44:11.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:11 smithi078 ceph-mon[63909]: pgmap v138: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.9 MiB/s wr, 514 op/s; 141/1548 objects degraded (9.109%) 2024-09-16T09:44:11.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 199886 10.05 MB/sec execute 391 sec latency 40.055 ms 2024-09-16T09:44:11.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:11 smithi088 ceph-mon[52758]: pgmap v138: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.9 MiB/s wr, 514 op/s; 141/1548 objects degraded (9.109%) 2024-09-16T09:44:12.416 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:12 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:44:12.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 200301 10.04 MB/sec execute 392 sec latency 59.536 ms 2024-09-16T09:44:12.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:12 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:44:13.441 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:13 smithi078 ceph-mon[63909]: pgmap v139: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.3 MiB/s wr, 394 op/s; 142/1539 objects degraded (9.227%) 2024-09-16T09:44:13.441 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:13 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:13.442 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:13 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:13.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 200570 10.03 MB/sec execute 393 sec latency 32.339 ms 2024-09-16T09:44:13.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:13 smithi088 ceph-mon[52758]: pgmap v139: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.3 MiB/s wr, 394 op/s; 142/1539 objects degraded (9.227%) 2024-09-16T09:44:13.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:13 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:13.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:13 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:14.458 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:14 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 142/1539 objects degraded (9.227%), 19 pgs degraded (PG_DEGRADED) 2024-09-16T09:44:14.458 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:14 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:14.459 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:14 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:14.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 201102 10.04 MB/sec execute 394 sec latency 37.237 ms 2024-09-16T09:44:14.710 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:44:14 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2[73557]: 2024-09-16T09:44:14.662+0000 7f745af44640 -1 osd.2 0 read_superblock omap replica is missing. 2024-09-16T09:44:14.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:14 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 142/1539 objects degraded (9.227%), 19 pgs degraded (PG_DEGRADED) 2024-09-16T09:44:14.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:14 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:14.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:14 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:15.459 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:15 smithi078 ceph-mon[63909]: pgmap v140: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.7 MiB/s wr, 742 op/s; 173/1737 objects degraded (9.960%) 2024-09-16T09:44:15.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 201737 10.05 MB/sec execute 395 sec latency 31.005 ms 2024-09-16T09:44:15.710 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:44:15 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2[73557]: 2024-09-16T09:44:15.668+0000 7f745af44640 -1 osd.2 101 log_to_monitors true 2024-09-16T09:44:15.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:15 smithi088 ceph-mon[52758]: pgmap v140: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.7 MiB/s wr, 742 op/s; 173/1737 objects degraded (9.960%) 2024-09-16T09:44:16.469 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:16 smithi078 ceph-mon[63909]: from='osd.2 [v2:172.21.15.78:6818/2257204018,v1:172.21.15.78:6819/2257204018]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-16T09:44:16.470 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:16 smithi078 ceph-mon[63909]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-16T09:44:16.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 202111 10.05 MB/sec execute 396 sec latency 87.264 ms 2024-09-16T09:44:16.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:16 smithi088 ceph-mon[52758]: from='osd.2 [v2:172.21.15.78:6818/2257204018,v1:172.21.15.78:6819/2257204018]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-16T09:44:16.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:16 smithi088 ceph-mon[52758]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-16T09:44:16.978 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:44:16 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2[73557]: 2024-09-16T09:44:16.889+0000 7f744ece6640 -1 osd.2 101 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T09:44:17.496 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:17 smithi078 ceph-mon[63909]: pgmap v141: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.9 MiB/s wr, 562 op/s; 182/1758 objects degraded (10.353%) 2024-09-16T09:44:17.497 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:17 smithi078 ceph-mon[63909]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-16T09:44:17.497 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:17 smithi078 ceph-mon[63909]: from='osd.2 [v2:172.21.15.78:6818/2257204018,v1:172.21.15.78:6819/2257204018]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]} : dispatch 2024-09-16T09:44:17.497 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:17 smithi078 ceph-mon[63909]: osdmap e104: 8 total, 7 up, 8 in 2024-09-16T09:44:17.497 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:17 smithi078 ceph-mon[63909]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]} : dispatch 2024-09-16T09:44:17.497 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:17 smithi078 ceph-mon[63909]: from='osd.2 ' entity='osd.2' 2024-09-16T09:44:17.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 202414 10.05 MB/sec execute 397 sec latency 141.097 ms 2024-09-16T09:44:17.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:17 smithi088 ceph-mon[52758]: pgmap v141: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.9 MiB/s wr, 562 op/s; 182/1758 objects degraded (10.353%) 2024-09-16T09:44:17.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:17 smithi088 ceph-mon[52758]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-16T09:44:17.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:17 smithi088 ceph-mon[52758]: from='osd.2 [v2:172.21.15.78:6818/2257204018,v1:172.21.15.78:6819/2257204018]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]} : dispatch 2024-09-16T09:44:17.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:17 smithi088 ceph-mon[52758]: osdmap e104: 8 total, 7 up, 8 in 2024-09-16T09:44:17.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:17 smithi088 ceph-mon[52758]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]} : dispatch 2024-09-16T09:44:17.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:17 smithi088 ceph-mon[52758]: from='osd.2 ' entity='osd.2' 2024-09-16T09:44:18.552 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:18 smithi078 ceph-mon[63909]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T09:44:18.553 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:18 smithi078 ceph-mon[63909]: osd.2 [v2:172.21.15.78:6818/2257204018,v1:172.21.15.78:6819/2257204018] boot 2024-09-16T09:44:18.553 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:18 smithi078 ceph-mon[63909]: osdmap e105: 8 total, 8 up, 8 in 2024-09-16T09:44:18.553 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:18 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-16T09:44:18.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 202667 10.04 MB/sec execute 398 sec latency 640.157 ms 2024-09-16T09:44:18.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:18 smithi088 ceph-mon[52758]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T09:44:18.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:18 smithi088 ceph-mon[52758]: osd.2 [v2:172.21.15.78:6818/2257204018,v1:172.21.15.78:6819/2257204018] boot 2024-09-16T09:44:18.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:18 smithi088 ceph-mon[52758]: osdmap e105: 8 total, 8 up, 8 in 2024-09-16T09:44:18.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:18 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-16T09:44:19.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 202704 10.02 MB/sec execute 399 sec latency 1640.186 ms 2024-09-16T09:44:19.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:19 smithi088 ceph-mon[52758]: pgmap v144: 97 pgs: 12 peering, 3 active+undersized, 10 active+undersized+degraded, 72 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 8.1 MiB/s wr, 821 op/s; 104/1656 objects degraded (6.280%) 2024-09-16T09:44:19.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:19 smithi088 ceph-mon[52758]: osdmap e106: 8 total, 8 up, 8 in 2024-09-16T09:44:19.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:19 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 104/1656 objects degraded (6.280%), 10 pgs degraded (PG_DEGRADED) 2024-09-16T09:44:19.803 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:19 smithi078 ceph-mon[63909]: pgmap v144: 97 pgs: 12 peering, 3 active+undersized, 10 active+undersized+degraded, 72 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 8.1 MiB/s wr, 821 op/s; 104/1656 objects degraded (6.280%) 2024-09-16T09:44:19.803 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:19 smithi078 ceph-mon[63909]: osdmap e106: 8 total, 8 up, 8 in 2024-09-16T09:44:19.804 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:19 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 104/1656 objects degraded (6.280%), 10 pgs degraded (PG_DEGRADED) 2024-09-16T09:44:20.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:20 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:20.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:20 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:20.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:20 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:44:20.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:20 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:44:20.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:20 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:20.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:20 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:20.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:20 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:44:20.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:20 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:44:20.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:20 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:44:20.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:20 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:44:20.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:20 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-16T09:44:20.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:20 smithi078 ceph-mon[63909]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-16T09:44:20.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:20 smithi078 ceph-mon[63909]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-09-16T09:44:20.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:20 smithi078 ceph-mon[63909]: osdmap e107: 8 total, 8 up, 8 in 2024-09-16T09:44:20.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 202929 10.02 MB/sec execute 400 sec latency 2026.256 ms 2024-09-16T09:44:20.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:20 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:20.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:20 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:20.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:20 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:44:20.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:20 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:44:20.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:20 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:20.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:20 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:20.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:20 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:44:20.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:20 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:44:20.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:20 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:44:20.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:20 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:44:20.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:20 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-16T09:44:20.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:20 smithi088 ceph-mon[52758]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-16T09:44:20.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:20 smithi088 ceph-mon[52758]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-09-16T09:44:20.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:20 smithi088 ceph-mon[52758]: osdmap e107: 8 total, 8 up, 8 in 2024-09-16T09:44:21.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 203140 10.01 MB/sec execute 401 sec latency 619.645 ms 2024-09-16T09:44:21.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:21 smithi088 ceph-mon[52758]: pgmap v147: 97 pgs: 16 peering, 2 active+undersized, 7 active+undersized+degraded, 72 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 6.6 MiB/s wr, 713 op/s; 65/1644 objects degraded (3.954%) 2024-09-16T09:44:21.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:21 smithi088 ceph-mon[52758]: osdmap e108: 8 total, 8 up, 8 in 2024-09-16T09:44:21.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:21 smithi078 ceph-mon[63909]: pgmap v147: 97 pgs: 16 peering, 2 active+undersized, 7 active+undersized+degraded, 72 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 6.6 MiB/s wr, 713 op/s; 65/1644 objects degraded (3.954%) 2024-09-16T09:44:21.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:21 smithi078 ceph-mon[63909]: osdmap e108: 8 total, 8 up, 8 in 2024-09-16T09:44:22.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 203337 10.00 MB/sec execute 402 sec latency 957.681 ms 2024-09-16T09:44:22.976 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:22 smithi078 ceph-mon[63909]: osdmap e109: 8 total, 8 up, 8 in 2024-09-16T09:44:22.977 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:22 smithi078 ceph-mon[63909]: pgmap v150: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 16 peering, 79 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 15 KiB/s rd, 2.8 MiB/s wr, 227 op/s; 23/1611 objects degraded (1.428%); 176 KiB/s, 8 objects/s recovering 2024-09-16T09:44:23.300 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:22 smithi088 ceph-mon[52758]: osdmap e109: 8 total, 8 up, 8 in 2024-09-16T09:44:23.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:22 smithi088 ceph-mon[52758]: pgmap v150: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 16 peering, 79 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 15 KiB/s rd, 2.8 MiB/s wr, 227 op/s; 23/1611 objects degraded (1.428%); 176 KiB/s, 8 objects/s recovering 2024-09-16T09:44:23.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 203581 9.99 MB/sec execute 403 sec latency 222.722 ms 2024-09-16T09:44:23.944 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:23 smithi078 ceph-mon[63909]: osdmap e110: 8 total, 8 up, 8 in 2024-09-16T09:44:24.122 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-16T09:44:24.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:23 smithi088 ceph-mon[52758]: osdmap e110: 8 total, 8 up, 8 in 2024-09-16T09:44:24.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 203876 9.97 MB/sec execute 404 sec latency 238.161 ms 2024-09-16T09:44:25.011 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T09:44:25.012 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (5m) 11s ago 16m 15.9M - 0.27.0 11f11916f8cd 3ebb6832fbcc 2024-09-16T09:44:25.012 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (2m) 11s ago 16m 6899k - 19.3.0-4882-g66e33d6a 261f2dc7b20f 72965213139c 2024-09-16T09:44:25.012 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi088 smithi088 running (2m) 2m ago 14m 6891k - 19.3.0-4882-g66e33d6a 261f2dc7b20f eac0e6855b1b 2024-09-16T09:44:25.012 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (5m) 11s ago 15m 46.5M - dad864ee21e9 4a91b97e29b1 2024-09-16T09:44:25.012 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.hiotnn smithi078 running (9m) 11s ago 9m 37.9M - 17.2.0 e1d6a67b021e c43db7767ab9 2024-09-16T09:44:25.012 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi088.dbunob smithi088 running (9m) 2m ago 9m 17.0M - 17.2.0 e1d6a67b021e d235f1bfbb05 2024-09-16T09:44:25.012 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rrfxge smithi078 *:8443,9283,8765 running (4m) 11s ago 17m 518M - 19.3.0-4882-g66e33d6a 261f2dc7b20f 0d25f2f30f1e 2024-09-16T09:44:25.012 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi088.ncxfdz smithi088 *:8443,9283,8765 running (4m) 2m ago 14m 453M - 19.3.0-4882-g66e33d6a 261f2dc7b20f e2d03f701acd 2024-09-16T09:44:25.012 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (3m) 11s ago 18m 53.9M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f f5a3a0fec757 2024-09-16T09:44:25.013 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi088 smithi088 running (3m) 2m ago 14m 37.7M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f 2e552147327a 2024-09-16T09:44:25.013 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.wawosh smithi078 *:2049 running (8m) 11s ago 8m 281M - 4.0 e1d6a67b021e 2544ed9d67f5 2024-09-16T09:44:25.013 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi088.bnccmw smithi088 *:2049 running (8m) 2m ago 8m 81.6M - 4.0 e1d6a67b021e e606adb67049 2024-09-16T09:44:25.013 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (5m) 11s ago 15m 10.6M - 1.7.0 72c9c2088986 895ac3ccb2f3 2024-09-16T09:44:25.013 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi088 smithi088 *:9100 running (5m) 2m ago 14m 10.9M - 1.7.0 72c9c2088986 c84a2e50de53 2024-09-16T09:44:25.013 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (118s) 11s ago 13m 314M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 85008b499cb7 2024-09-16T09:44:25.013 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (71s) 11s ago 13m 228M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 12c137fcba70 2024-09-16T09:44:25.013 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (21s) 11s ago 12m 11.6M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 23a8b69c5c64 2024-09-16T09:44:25.013 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (12m) 11s ago 12m 361M 1936M 17.2.0 e1d6a67b021e db62172fe90c 2024-09-16T09:44:25.013 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi088 running (12m) 2m ago 12m 505M 2704M 17.2.0 e1d6a67b021e 6f4e80fe127a 2024-09-16T09:44:25.013 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi088 running (11m) 2m ago 11m 322M 2704M 17.2.0 e1d6a67b021e 73e51edb7756 2024-09-16T09:44:25.013 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi088 running (11m) 2m ago 11m 503M 2704M 17.2.0 e1d6a67b021e 7c1459b1a2d9 2024-09-16T09:44:25.014 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi088 running (10m) 2m ago 10m 398M 2704M 17.2.0 e1d6a67b021e 37f619c2ddf9 2024-09-16T09:44:25.014 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (4m) 11s ago 14m 53.1M - 2.51.0 1d3b7f56885b 4c956ead9125 2024-09-16T09:44:25.040 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:24 smithi078 ceph-mon[63909]: osdmap e111: 8 total, 8 up, 8 in 2024-09-16T09:44:25.041 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:24 smithi078 ceph-mon[63909]: pgmap v153: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 93 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 15 KiB/s rd, 4.0 MiB/s wr, 350 op/s; 60/1629 objects degraded (3.683%); 1.2 MiB/s, 51 objects/s recovering 2024-09-16T09:44:25.041 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:24 smithi078 ceph-mon[63909]: from='client.34280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:44:25.041 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:24 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 23/1611 objects degraded (1.428%), 1 pg degraded (PG_DEGRADED) 2024-09-16T09:44:25.041 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:24 smithi078 ceph-mon[63909]: from='client.34284 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:44:25.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:24 smithi088 ceph-mon[52758]: osdmap e111: 8 total, 8 up, 8 in 2024-09-16T09:44:25.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:24 smithi088 ceph-mon[52758]: pgmap v153: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 93 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 15 KiB/s rd, 4.0 MiB/s wr, 350 op/s; 60/1629 objects degraded (3.683%); 1.2 MiB/s, 51 objects/s recovering 2024-09-16T09:44:25.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:24 smithi088 ceph-mon[52758]: from='client.34280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:44:25.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:24 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 23/1611 objects degraded (1.428%), 1 pg degraded (PG_DEGRADED) 2024-09-16T09:44:25.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:24 smithi088 ceph-mon[52758]: from='client.34284 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:44:25.560 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:44:25.560 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-16T09:44:25.560 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:44:25.560 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:44:25.560 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-16T09:44:25.560 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:44:25.561 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:44:25.561 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-16T09:44:25.561 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-16T09:44:25.561 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 3 2024-09-16T09:44:25.561 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:44:25.561 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-16T09:44:25.561 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:44:25.561 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:44:25.561 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-16T09:44:25.561 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-16T09:44:25.561 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 7 2024-09-16T09:44:25.561 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-16T09:44:25.562 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:44:25.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 204042 9.96 MB/sec execute 405 sec latency 647.986 ms 2024-09-16T09:44:25.975 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:44:25.976 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f", 2024-09-16T09:44:25.976 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-16T09:44:25.976 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T09:44:25.976 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-16T09:44:25.976 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-09-16T09:44:25.976 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-16T09:44:25.976 INFO:teuthology.orchestra.run.smithi078.stdout: "mon" 2024-09-16T09:44:25.976 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-16T09:44:25.976 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "9/23 daemons upgraded", 2024-09-16T09:44:25.976 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-09-16T09:44:25.976 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-16T09:44:25.976 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:44:26.064 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:25 smithi078 ceph-mon[63909]: osdmap e112: 8 total, 8 up, 8 in 2024-09-16T09:44:26.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:25 smithi078 ceph-mon[63909]: from='client.34288 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:44:26.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:25 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/3926769682' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:44:26.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:25 smithi088 ceph-mon[52758]: osdmap e112: 8 total, 8 up, 8 in 2024-09-16T09:44:26.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:25 smithi088 ceph-mon[52758]: from='client.34288 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:44:26.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:25 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/3926769682' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:44:26.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 204233 9.95 MB/sec execute 406 sec latency 57.024 ms 2024-09-16T09:44:26.572 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_WARN Degraded data redundancy: 60/1629 objects degraded (3.683%), 3 pgs degraded 2024-09-16T09:44:26.572 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 60/1629 objects degraded (3.683%), 3 pgs degraded 2024-09-16T09:44:26.572 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.4 is active+recovery_wait+undersized+degraded+remapped, acting [1,5] 2024-09-16T09:44:26.572 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.5 is active+recovering+undersized+degraded+remapped, acting [5,3] 2024-09-16T09:44:26.572 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1a is active+recovery_wait+undersized+degraded+remapped, acting [4,1] 2024-09-16T09:44:27.100 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:26 smithi078 ceph-mon[63909]: from='client.34296 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:44:27.101 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:26 smithi078 ceph-mon[63909]: pgmap v155: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.1 MiB/s wr, 339 op/s; 37/1635 objects degraded (2.263%); 1.1 MiB/s, 47 objects/s recovering 2024-09-16T09:44:27.101 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:26 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/3526781382' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:44:27.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:26 smithi088 ceph-mon[52758]: from='client.34296 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:44:27.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:26 smithi088 ceph-mon[52758]: pgmap v155: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.1 MiB/s wr, 339 op/s; 37/1635 objects degraded (2.263%); 1.1 MiB/s, 47 objects/s recovering 2024-09-16T09:44:27.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:26 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/3526781382' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:44:27.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 204546 9.95 MB/sec execute 407 sec latency 73.011 ms 2024-09-16T09:44:28.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:28 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:28.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:28 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:44:28.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 204816 9.93 MB/sec execute 408 sec latency 93.214 ms 2024-09-16T09:44:28.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:28 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:28.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:28 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:44:29.430 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:29 smithi078 ceph-mon[63909]: pgmap v156: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 488 op/s; 741 KiB/s, 39 objects/s recovering 2024-09-16T09:44:29.431 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:29 smithi078 ceph-mon[63909]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 37/1635 objects degraded (2.263%), 2 pgs degraded) 2024-09-16T09:44:29.431 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:29 smithi078 ceph-mon[63909]: Cluster is now healthy 2024-09-16T09:44:29.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 205133 9.92 MB/sec execute 409 sec latency 55.361 ms 2024-09-16T09:44:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:29 smithi088 ceph-mon[52758]: pgmap v156: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 488 op/s; 741 KiB/s, 39 objects/s recovering 2024-09-16T09:44:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:29 smithi088 ceph-mon[52758]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 37/1635 objects degraded (2.263%), 2 pgs degraded) 2024-09-16T09:44:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:29 smithi088 ceph-mon[52758]: Cluster is now healthy 2024-09-16T09:44:30.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 205361 9.91 MB/sec execute 410 sec latency 56.243 ms 2024-09-16T09:44:31.461 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:31 smithi078 ceph-mon[63909]: pgmap v157: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.2 MiB/s wr, 449 op/s; 631 KiB/s, 28 objects/s recovering 2024-09-16T09:44:31.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 205676 9.92 MB/sec execute 411 sec latency 51.083 ms 2024-09-16T09:44:31.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:31 smithi088 ceph-mon[52758]: pgmap v157: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.2 MiB/s wr, 449 op/s; 631 KiB/s, 28 objects/s recovering 2024-09-16T09:44:32.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 206054 9.93 MB/sec execute 412 sec latency 46.169 ms 2024-09-16T09:44:33.505 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:33 smithi078 ceph-mon[63909]: pgmap v158: 97 pgs: 97 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 2.2 MiB/s wr, 253 op/s; 20 KiB/s, 8 objects/s recovering 2024-09-16T09:44:33.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 206365 9.93 MB/sec execute 413 sec latency 76.406 ms 2024-09-16T09:44:33.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:33 smithi088 ceph-mon[52758]: pgmap v158: 97 pgs: 97 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 2.2 MiB/s wr, 253 op/s; 20 KiB/s, 8 objects/s recovering 2024-09-16T09:44:34.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 206592 9.92 MB/sec execute 414 sec latency 44.016 ms 2024-09-16T09:44:35.549 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:35 smithi078 ceph-mon[63909]: pgmap v159: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.7 MiB/s wr, 333 op/s; 16 KiB/s, 6 objects/s recovering 2024-09-16T09:44:35.549 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:35 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-16T09:44:35.549 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:35 smithi078 ceph-mon[63909]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-16T09:44:35.549 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:35 smithi078 ceph-mon[63909]: Upgrade: osd.3 is safe to restart 2024-09-16T09:44:35.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 206893 9.91 MB/sec execute 415 sec latency 47.719 ms 2024-09-16T09:44:35.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:35 smithi088 ceph-mon[52758]: pgmap v159: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.7 MiB/s wr, 333 op/s; 16 KiB/s, 6 objects/s recovering 2024-09-16T09:44:35.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:35 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-16T09:44:35.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:35 smithi088 ceph-mon[52758]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-16T09:44:35.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:35 smithi088 ceph-mon[52758]: Upgrade: osd.3 is safe to restart 2024-09-16T09:44:36.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 207218 9.91 MB/sec execute 416 sec latency 48.281 ms 2024-09-16T09:44:37.338 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:37 smithi078 ceph-mon[63909]: pgmap v160: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.9 MiB/s wr, 408 op/s; 14 KiB/s, 6 objects/s recovering 2024-09-16T09:44:37.339 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:37 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:37.339 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:37 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-16T09:44:37.339 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:37 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:44:37.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 207527 9.90 MB/sec execute 417 sec latency 57.901 ms 2024-09-16T09:44:37.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:37 smithi088 ceph-mon[52758]: pgmap v160: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.9 MiB/s wr, 408 op/s; 14 KiB/s, 6 objects/s recovering 2024-09-16T09:44:37.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:37 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:37.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:37 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-16T09:44:37.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:37 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:44:38.374 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:38 smithi078 ceph-mon[63909]: Upgrade: Updating osd.3 2024-09-16T09:44:38.374 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:38 smithi078 ceph-mon[63909]: Deploying daemon osd.3 on smithi078 2024-09-16T09:44:38.374 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:38 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:38.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 207852 9.90 MB/sec execute 418 sec latency 59.797 ms 2024-09-16T09:44:38.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:38 smithi088 ceph-mon[52758]: Upgrade: Updating osd.3 2024-09-16T09:44:38.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:38 smithi088 ceph-mon[52758]: Deploying daemon osd.3 on smithi078 2024-09-16T09:44:38.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:38 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:39.383 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:39 smithi078 ceph-mon[63909]: pgmap v161: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.3 MiB/s wr, 473 op/s; 0 B/s, 3 objects/s recovering 2024-09-16T09:44:39.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 208189 9.89 MB/sec execute 419 sec latency 78.305 ms 2024-09-16T09:44:39.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:39 smithi088 ceph-mon[52758]: pgmap v161: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.3 MiB/s wr, 473 op/s; 0 B/s, 3 objects/s recovering 2024-09-16T09:44:40.568 INFO:teuthology.orchestra.run.smithi078.stdout: 5 208537 9.88 MB/sec execute 420 sec latency 54.960 ms 2024-09-16T09:44:41.439 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:41 smithi078 ceph-mon[63909]: pgmap v162: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.4 MiB/s wr, 401 op/s 2024-09-16T09:44:41.568 INFO:teuthology.orchestra.run.smithi078.stdout: 5 208860 9.88 MB/sec execute 421 sec latency 45.790 ms 2024-09-16T09:44:41.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:41 smithi088 ceph-mon[52758]: pgmap v162: 97 pgs: 97 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.4 MiB/s wr, 401 op/s 2024-09-16T09:44:42.475 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:42 smithi078 systemd[1]: Stopping Ceph osd.3 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:44:42.567 INFO:teuthology.orchestra.run.smithi078.stdout: 5 209210 9.90 MB/sec execute 422 sec latency 52.446 ms 2024-09-16T09:44:42.741 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:42 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3[45107]: 2024-09-16T09:44:42.712+0000 7f591b1bc700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T09:44:42.741 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:42 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3[45107]: 2024-09-16T09:44:42.712+0000 7f591b1bc700 -1 osd.3 112 *** Got signal Terminated *** 2024-09-16T09:44:42.741 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:42 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3[45107]: 2024-09-16T09:44:42.712+0000 7f591b1bc700 -1 osd.3 112 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T09:44:43.503 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:43 smithi078 ceph-mon[63909]: pgmap v163: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.0 MiB/s wr, 384 op/s 2024-09-16T09:44:43.504 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:43 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:43.504 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:43 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:44:43.504 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:43 smithi078 ceph-mon[63909]: osd.3 marked itself down and dead 2024-09-16T09:44:43.568 INFO:teuthology.orchestra.run.smithi078.stdout: 5 209565 9.89 MB/sec execute 423 sec latency 163.156 ms 2024-09-16T09:44:43.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:43 smithi088 ceph-mon[52758]: pgmap v163: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.0 MiB/s wr, 384 op/s 2024-09-16T09:44:43.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:43 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:43.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:43 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:44:43.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:43 smithi088 ceph-mon[52758]: osd.3 marked itself down and dead 2024-09-16T09:44:44.300 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:44 smithi078 podman[75448]: 2024-09-16 09:44:44.104076789 +0000 UTC m=+1.527410549 container died db62172fe90c8132722fb89c403e71eacf71526dc55ddba07330d515ceb19658 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3, build-date=2022-05-03T08:36:31.336870, name=centos-stream, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , GIT_CLEAN=True, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, ceph=True, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-16T09:44:44.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:44 smithi078 ceph-mon[63909]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T09:44:44.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:44 smithi078 ceph-mon[63909]: osdmap e113: 8 total, 7 up, 8 in 2024-09-16T09:44:44.568 INFO:teuthology.orchestra.run.smithi078.stdout: 5 209682 9.87 MB/sec execute 424 sec latency 1018.502 ms 2024-09-16T09:44:44.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:44 smithi088 ceph-mon[52758]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T09:44:44.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:44 smithi088 ceph-mon[52758]: osdmap e113: 8 total, 7 up, 8 in 2024-09-16T09:44:44.817 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:44 smithi078 podman[75448]: 2024-09-16 09:44:44.653410921 +0000 UTC m=+2.076744676 container cleanup db62172fe90c8132722fb89c403e71eacf71526dc55ddba07330d515ceb19658 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , GIT_CLEAN=True, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, release=754, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64) 2024-09-16T09:44:44.817 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:44 smithi078 bash[75448]: ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3 2024-09-16T09:44:45.086 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:44 smithi078 podman[75464]: 2024-09-16 09:44:44.960046742 +0000 UTC m=+0.849292129 container remove db62172fe90c8132722fb89c403e71eacf71526dc55ddba07330d515ceb19658 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, RELEASE=HEAD, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., name=centos-stream, version=8, build-date=2022-05-03T08:36:31.336870, vcs-type=git, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, distribution-scope=public, ceph=True, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-09-16T09:44:45.341 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:45 smithi078 ceph-mon[63909]: pgmap v165: 97 pgs: 9 peering, 13 stale+active+clean, 75 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.9 MiB/s wr, 549 op/s 2024-09-16T09:44:45.342 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:45 smithi078 ceph-mon[63909]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-16T09:44:45.342 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:45 smithi078 ceph-mon[63909]: osdmap e114: 8 total, 7 up, 8 in 2024-09-16T09:44:45.568 INFO:teuthology.orchestra.run.smithi078.stdout: 5 209997 9.88 MB/sec execute 425 sec latency 98.683 ms 2024-09-16T09:44:45.595 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:45 smithi078 podman[75551]: 2024-09-16 09:44:45.444225173 +0000 UTC m=+0.023568784 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:44:45.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:45 smithi088 ceph-mon[52758]: pgmap v165: 97 pgs: 9 peering, 13 stale+active+clean, 75 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.9 MiB/s wr, 549 op/s 2024-09-16T09:44:45.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:45 smithi088 ceph-mon[52758]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-16T09:44:45.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:45 smithi088 ceph-mon[52758]: osdmap e114: 8 total, 7 up, 8 in 2024-09-16T09:44:45.861 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:45 smithi078 podman[75551]: 2024-09-16 09:44:45.77476979 +0000 UTC m=+0.354113389 container create a5ae1cc9eb74c72a18e3de19b4d4c5c3bb21103ecdbbf300a4ea4215cce01831 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-16T09:44:46.363 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:46 smithi078 podman[75551]: 2024-09-16 09:44:46.204897587 +0000 UTC m=+0.784241207 container init a5ae1cc9eb74c72a18e3de19b4d4c5c3bb21103ecdbbf300a4ea4215cce01831 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-16T09:44:46.364 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:46 smithi078 podman[75551]: 2024-09-16 09:44:46.215085758 +0000 UTC m=+0.794429355 container start a5ae1cc9eb74c72a18e3de19b4d4c5c3bb21103ecdbbf300a4ea4215cce01831 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3-deactivate, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-16T09:44:46.364 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:46 smithi078 podman[75551]: 2024-09-16 09:44:46.321445123 +0000 UTC m=+0.900788745 container attach a5ae1cc9eb74c72a18e3de19b4d4c5c3bb21103ecdbbf300a4ea4215cce01831 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3-deactivate, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-16T09:44:46.568 INFO:teuthology.orchestra.run.smithi078.stdout: 5 210181 9.87 MB/sec execute 426 sec latency 51.301 ms 2024-09-16T09:44:46.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:46 smithi078 ceph-mon[63909]: Health check failed: Degraded data redundancy: 30/1617 objects degraded (1.855%), 6 pgs degraded (PG_DEGRADED) 2024-09-16T09:44:46.618 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:46 smithi078 podman[75551]: 2024-09-16 09:44:46.532840667 +0000 UTC m=+1.112184278 container died a5ae1cc9eb74c72a18e3de19b4d4c5c3bb21103ecdbbf300a4ea4215cce01831 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-16T09:44:46.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:46 smithi088 ceph-mon[52758]: Health check failed: Degraded data redundancy: 30/1617 objects degraded (1.855%), 6 pgs degraded (PG_DEGRADED) 2024-09-16T09:44:47.389 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:47 smithi078 ceph-mon[63909]: pgmap v167: 97 pgs: 3 active+undersized, 9 peering, 10 stale+active+clean, 6 active+undersized+degraded, 69 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.4 MiB/s wr, 393 op/s; 30/1617 objects degraded (1.855%) 2024-09-16T09:44:47.568 INFO:teuthology.orchestra.run.smithi078.stdout: 5 210424 9.86 MB/sec execute 427 sec latency 68.597 ms 2024-09-16T09:44:47.652 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:47 smithi078 podman[75573]: 2024-09-16 09:44:47.411895684 +0000 UTC m=+0.872515946 container remove a5ae1cc9eb74c72a18e3de19b4d4c5c3bb21103ecdbbf300a4ea4215cce01831 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-16T09:44:47.652 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:47 smithi078 systemd[1]: ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.3.service: Deactivated successfully. 2024-09-16T09:44:47.652 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:47 smithi078 systemd[1]: Stopped Ceph osd.3 for a70c765e-740d-11ef-bceb-c7b262605968. 2024-09-16T09:44:47.652 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:47 smithi078 systemd[1]: ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.3.service: Consumed 4min 9.352s CPU time. 2024-09-16T09:44:47.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:47 smithi088 ceph-mon[52758]: pgmap v167: 97 pgs: 3 active+undersized, 9 peering, 10 stale+active+clean, 6 active+undersized+degraded, 69 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.4 MiB/s wr, 393 op/s; 30/1617 objects degraded (1.855%) 2024-09-16T09:44:47.902 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:47 smithi078 systemd[1]: Starting Ceph osd.3 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:44:48.157 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:47 smithi078 podman[75668]: 2024-09-16 09:44:47.886983766 +0000 UTC m=+0.021961251 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:44:48.158 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:48 smithi078 podman[75668]: 2024-09-16 09:44:48.062042031 +0000 UTC m=+0.197019500 container create cf6046d9929af601ff6458521ed449bc456240207604c700118c35959943f3bc (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-16T09:44:48.568 INFO:teuthology.orchestra.run.smithi078.stdout: 5 210717 9.85 MB/sec execute 428 sec latency 63.082 ms 2024-09-16T09:44:48.681 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:48 smithi078 podman[75668]: 2024-09-16 09:44:48.510875963 +0000 UTC m=+0.645853446 container init cf6046d9929af601ff6458521ed449bc456240207604c700118c35959943f3bc (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True) 2024-09-16T09:44:48.682 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:48 smithi078 podman[75668]: 2024-09-16 09:44:48.523886493 +0000 UTC m=+0.658863965 container start cf6046d9929af601ff6458521ed449bc456240207604c700118c35959943f3bc (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731) 2024-09-16T09:44:48.938 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:48 smithi078 podman[75668]: 2024-09-16 09:44:48.697071932 +0000 UTC m=+0.832049402 container attach cf6046d9929af601ff6458521ed449bc456240207604c700118c35959943f3bc (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3-activate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-16T09:44:48.939 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:48 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3-activate[75682]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:44:48.939 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:48 smithi078 bash[75668]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:44:48.939 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:48 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3-activate[75682]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:44:48.939 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:48 smithi078 bash[75668]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:44:49.443 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:49 smithi078 ceph-mon[63909]: pgmap v168: 97 pgs: 10 active+undersized, 9 peering, 5 stale+active+clean, 16 active+undersized+degraded, 57 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.2 MiB/s wr, 480 op/s; 110/1632 objects degraded (6.740%) 2024-09-16T09:44:49.444 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:49 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3-activate[75682]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-16T09:44:49.444 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:49 smithi078 bash[75668]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-16T09:44:49.444 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:49 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3-activate[75682]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-16T09:44:49.444 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:49 smithi078 bash[75668]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-16T09:44:49.568 INFO:teuthology.orchestra.run.smithi078.stdout: 5 211064 9.84 MB/sec execute 429 sec latency 39.895 ms 2024-09-16T09:44:49.713 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:49 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3-activate[75682]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-16T09:44:49.713 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:49 smithi078 bash[75668]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-16T09:44:49.713 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:49 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3-activate[75682]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-16T09:44:49.713 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:49 smithi078 bash[75668]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-16T09:44:49.713 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:49 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3-activate[75682]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-16T09:44:49.713 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:49 smithi078 bash[75668]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-16T09:44:49.713 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:49 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3-activate[75682]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-16T09:44:49.713 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:49 smithi078 bash[75668]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-16T09:44:49.713 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:49 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3-activate[75682]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-16T09:44:49.713 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:49 smithi078 bash[75668]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-16T09:44:49.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:49 smithi088 ceph-mon[52758]: pgmap v168: 97 pgs: 10 active+undersized, 9 peering, 5 stale+active+clean, 16 active+undersized+degraded, 57 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.2 MiB/s wr, 480 op/s; 110/1632 objects degraded (6.740%) 2024-09-16T09:44:49.963 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:49 smithi078 podman[75829]: 2024-09-16 09:44:49.785340615 +0000 UTC m=+0.028844166 container died cf6046d9929af601ff6458521ed449bc456240207604c700118c35959943f3bc (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-16T09:44:50.465 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:50 smithi078 ceph-mon[63909]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-16T09:44:50.568 INFO:teuthology.orchestra.run.smithi078.stdout: 5 211416 9.85 MB/sec execute 430 sec latency 43.300 ms 2024-09-16T09:44:50.716 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:50 smithi078 podman[75829]: 2024-09-16 09:44:50.63959883 +0000 UTC m=+0.883102379 container remove cf6046d9929af601ff6458521ed449bc456240207604c700118c35959943f3bc (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3-activate, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-16T09:44:50.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:50 smithi088 ceph-mon[52758]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-16T09:44:50.971 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:50 smithi078 podman[75884]: 2024-09-16 09:44:50.824532723 +0000 UTC m=+0.031607547 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:44:50.972 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:50 smithi078 podman[75884]: 2024-09-16 09:44:50.945680578 +0000 UTC m=+0.152755395 container create 00c9aa84d41d4002000786c60f19fe92d7e21fdd3e0fcd1dd7153959678f6634 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6) 2024-09-16T09:44:51.481 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:51 smithi078 ceph-mon[63909]: pgmap v169: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.1 MiB/s wr, 508 op/s; 214/1653 objects degraded (12.946%) 2024-09-16T09:44:51.481 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:51 smithi078 podman[75884]: 2024-09-16 09:44:51.459655302 +0000 UTC m=+0.666730120 container init 00c9aa84d41d4002000786c60f19fe92d7e21fdd3e0fcd1dd7153959678f6634 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-16T09:44:51.481 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:51 smithi078 podman[75884]: 2024-09-16 09:44:51.467665982 +0000 UTC m=+0.674740798 container start 00c9aa84d41d4002000786c60f19fe92d7e21fdd3e0fcd1dd7153959678f6634 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-16T09:44:51.568 INFO:teuthology.orchestra.run.smithi078.stdout: 5 211845 9.85 MB/sec execute 431 sec latency 52.962 ms 2024-09-16T09:44:51.734 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:51 smithi078 ceph-osd[75901]: -- 172.21.15.78:0/387314680 <== mon.0 v2:172.21.15.78:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f4b08d41e0 con 0x55f4b08ce000 2024-09-16T09:44:51.734 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:51 smithi078 bash[75884]: 00c9aa84d41d4002000786c60f19fe92d7e21fdd3e0fcd1dd7153959678f6634 2024-09-16T09:44:51.734 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:51 smithi078 systemd[1]: Started Ceph osd.3 for a70c765e-740d-11ef-bceb-c7b262605968. 2024-09-16T09:44:51.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:51 smithi088 ceph-mon[52758]: pgmap v169: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.1 MiB/s wr, 508 op/s; 214/1653 objects degraded (12.946%) 2024-09-16T09:44:52.568 INFO:teuthology.orchestra.run.smithi078.stdout: 5 212279 9.84 MB/sec execute 432 sec latency 51.765 ms 2024-09-16T09:44:52.757 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:52 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:52.757 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:52 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:52.757 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:52 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:44:53.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:52 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:53.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:52 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:53.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:52 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:44:53.568 INFO:teuthology.orchestra.run.smithi078.stdout: 5 212481 9.83 MB/sec execute 433 sec latency 37.919 ms 2024-09-16T09:44:53.796 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:53 smithi078 ceph-mon[63909]: pgmap v170: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.2 MiB/s wr, 316 op/s; 212/1650 objects degraded (12.848%) 2024-09-16T09:44:53.797 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:53 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 212/1650 objects degraded (12.848%), 27 pgs degraded (PG_DEGRADED) 2024-09-16T09:44:53.997 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:44:53 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3[75896]: 2024-09-16T09:44:53.812+0000 7f00f81df640 -1 Falling back to public interface 2024-09-16T09:44:54.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:53 smithi088 ceph-mon[52758]: pgmap v170: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.2 MiB/s wr, 316 op/s; 212/1650 objects degraded (12.848%) 2024-09-16T09:44:54.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:53 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 212/1650 objects degraded (12.848%), 27 pgs degraded (PG_DEGRADED) 2024-09-16T09:44:54.568 INFO:teuthology.orchestra.run.smithi078.stdout: 5 212788 9.84 MB/sec execute 434 sec latency 68.991 ms 2024-09-16T09:44:55.569 INFO:teuthology.orchestra.run.smithi078.stdout: 5 213208 9.84 MB/sec execute 435 sec latency 94.594 ms 2024-09-16T09:44:55.767 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:55 smithi078 ceph-mon[63909]: pgmap v171: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.3 MiB/s wr, 400 op/s; 206/1626 objects degraded (12.669%) 2024-09-16T09:44:56.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:55 smithi088 ceph-mon[52758]: pgmap v171: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.3 MiB/s wr, 400 op/s; 206/1626 objects degraded (12.669%) 2024-09-16T09:44:56.568 INFO:teuthology.orchestra.run.smithi078.stdout: 5 213504 9.84 MB/sec execute 436 sec latency 55.616 ms 2024-09-16T09:44:57.053 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-16T09:44:57.568 INFO:teuthology.orchestra.run.smithi078.stdout: 5 213684 9.84 MB/sec execute 437 sec latency 79.494 ms 2024-09-16T09:44:57.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:57 smithi078 ceph-mon[63909]: pgmap v172: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.0 MiB/s wr, 398 op/s; 211/1635 objects degraded (12.905%) 2024-09-16T09:44:57.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:57 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:57.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:57 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:44:57.979 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T09:44:57.979 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (6m) 44s ago 17m 15.9M - 0.27.0 11f11916f8cd 3ebb6832fbcc 2024-09-16T09:44:57.980 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (3m) 44s ago 17m 6899k - 19.3.0-4882-g66e33d6a 261f2dc7b20f 72965213139c 2024-09-16T09:44:57.980 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi088 smithi088 running (3m) 2m ago 15m 6891k - 19.3.0-4882-g66e33d6a 261f2dc7b20f eac0e6855b1b 2024-09-16T09:44:57.980 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (6m) 44s ago 15m 46.5M - dad864ee21e9 4a91b97e29b1 2024-09-16T09:44:57.980 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.hiotnn smithi078 running (9m) 44s ago 9m 37.9M - 17.2.0 e1d6a67b021e c43db7767ab9 2024-09-16T09:44:57.980 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi088.dbunob smithi088 running (9m) 2m ago 9m 17.0M - 17.2.0 e1d6a67b021e d235f1bfbb05 2024-09-16T09:44:57.980 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rrfxge smithi078 *:8443,9283,8765 running (5m) 44s ago 18m 518M - 19.3.0-4882-g66e33d6a 261f2dc7b20f 0d25f2f30f1e 2024-09-16T09:44:57.980 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi088.ncxfdz smithi088 *:8443,9283,8765 running (4m) 2m ago 15m 453M - 19.3.0-4882-g66e33d6a 261f2dc7b20f e2d03f701acd 2024-09-16T09:44:57.980 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (4m) 44s ago 18m 53.9M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f f5a3a0fec757 2024-09-16T09:44:57.980 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi088 smithi088 running (3m) 2m ago 15m 37.7M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f 2e552147327a 2024-09-16T09:44:57.980 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.wawosh smithi078 *:2049 running (9m) 44s ago 9m 281M - 4.0 e1d6a67b021e 2544ed9d67f5 2024-09-16T09:44:57.980 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi088.bnccmw smithi088 *:2049 running (9m) 2m ago 9m 81.6M - 4.0 e1d6a67b021e e606adb67049 2024-09-16T09:44:57.980 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (6m) 44s ago 15m 10.6M - 1.7.0 72c9c2088986 895ac3ccb2f3 2024-09-16T09:44:57.981 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi088 smithi088 *:9100 running (5m) 2m ago 15m 10.9M - 1.7.0 72c9c2088986 c84a2e50de53 2024-09-16T09:44:57.981 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (2m) 44s ago 14m 314M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 85008b499cb7 2024-09-16T09:44:57.981 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (104s) 44s ago 13m 228M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 12c137fcba70 2024-09-16T09:44:57.981 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (54s) 44s ago 13m 11.6M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 23a8b69c5c64 2024-09-16T09:44:57.981 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 starting - - - 1936M 2024-09-16T09:44:57.981 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi088 running (12m) 2m ago 12m 505M 2704M 17.2.0 e1d6a67b021e 6f4e80fe127a 2024-09-16T09:44:57.981 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi088 running (12m) 2m ago 12m 322M 2704M 17.2.0 e1d6a67b021e 73e51edb7756 2024-09-16T09:44:57.981 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi088 running (11m) 2m ago 11m 503M 2704M 17.2.0 e1d6a67b021e 7c1459b1a2d9 2024-09-16T09:44:57.981 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi088 running (11m) 2m ago 11m 398M 2704M 17.2.0 e1d6a67b021e 37f619c2ddf9 2024-09-16T09:44:57.981 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (4m) 44s ago 14m 53.1M - 2.51.0 1d3b7f56885b 4c956ead9125 2024-09-16T09:44:58.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:57 smithi088 ceph-mon[52758]: pgmap v172: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.0 MiB/s wr, 398 op/s; 211/1635 objects degraded (12.905%) 2024-09-16T09:44:58.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:57 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:44:58.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:57 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:44:58.552 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:44:58.552 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-16T09:44:58.552 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:44:58.552 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:44:58.552 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-16T09:44:58.552 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:44:58.552 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:44:58.552 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-16T09:44:58.553 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-16T09:44:58.553 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 3 2024-09-16T09:44:58.553 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:44:58.553 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-16T09:44:58.553 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:44:58.553 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:44:58.553 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-16T09:44:58.553 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-16T09:44:58.553 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 7 2024-09-16T09:44:58.553 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-16T09:44:58.553 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:44:58.568 INFO:teuthology.orchestra.run.smithi078.stdout: 5 213935 9.83 MB/sec execute 438 sec latency 57.283 ms 2024-09-16T09:44:58.909 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:58 smithi078 ceph-mon[63909]: from='client.34306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:44:58.909 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:58 smithi078 ceph-mon[63909]: from='client.34310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:44:58.910 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:58 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/879969903' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:44:59.025 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:44:59.025 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f", 2024-09-16T09:44:59.025 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-16T09:44:59.025 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T09:44:59.026 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-16T09:44:59.026 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-09-16T09:44:59.026 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-16T09:44:59.026 INFO:teuthology.orchestra.run.smithi078.stdout: "mon" 2024-09-16T09:44:59.026 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-16T09:44:59.026 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "9/23 daemons upgraded", 2024-09-16T09:44:59.026 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-09-16T09:44:59.026 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-16T09:44:59.026 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:44:59.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:58 smithi088 ceph-mon[52758]: from='client.34306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:44:59.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:58 smithi088 ceph-mon[52758]: from='client.34310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:44:59.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:58 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/879969903' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:44:59.569 INFO:teuthology.orchestra.run.smithi078.stdout: 5 214194 9.82 MB/sec execute 439 sec latency 63.092 ms 2024-09-16T09:44:59.633 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 204/1626 objects degraded (12.546%), 27 pgs degraded 2024-09-16T09:44:59.633 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-16T09:44:59.633 INFO:teuthology.orchestra.run.smithi078.stdout: osd.3 (root=default,host=smithi078) is down 2024-09-16T09:44:59.633 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 204/1626 objects degraded (12.546%), 27 pgs degraded 2024-09-16T09:44:59.633 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1 is active+undersized+degraded, acting [2,0] 2024-09-16T09:44:59.633 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.9 is active+undersized+degraded, acting [1,7] 2024-09-16T09:44:59.633 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.a is active+undersized+degraded, acting [1,7] 2024-09-16T09:44:59.633 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.d is active+undersized+degraded, acting [1,4] 2024-09-16T09:44:59.633 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.e is active+undersized+degraded, acting [2,7] 2024-09-16T09:44:59.633 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.12 is active+undersized+degraded, acting [5,7] 2024-09-16T09:44:59.633 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.13 is active+undersized+degraded, acting [0,4] 2024-09-16T09:44:59.633 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.14 is active+undersized+degraded, acting [6,5] 2024-09-16T09:44:59.634 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.19 is active+undersized+degraded, acting [6,0] 2024-09-16T09:44:59.634 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1b is active+undersized+degraded, acting [7,6] 2024-09-16T09:44:59.634 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1e is active+undersized+degraded, acting [0,5] 2024-09-16T09:44:59.634 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1 is active+undersized+degraded, acting [0,4] 2024-09-16T09:44:59.634 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.2 is active+undersized+degraded, acting [5,6] 2024-09-16T09:44:59.634 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.5 is active+undersized+degraded, acting [5,2] 2024-09-16T09:44:59.634 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.7 is active+undersized+degraded, acting [7,0] 2024-09-16T09:44:59.634 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.8 is active+undersized+degraded, acting [1,7] 2024-09-16T09:44:59.634 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.b is active+undersized+degraded, acting [0,4] 2024-09-16T09:44:59.634 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.c is active+undersized+degraded, acting [5,6] 2024-09-16T09:44:59.634 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.12 is active+undersized+degraded, acting [0,7] 2024-09-16T09:44:59.634 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.15 is active+undersized+degraded, acting [7,4] 2024-09-16T09:44:59.634 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.17 is active+undersized+degraded, acting [0,5] 2024-09-16T09:44:59.635 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.18 is active+undersized+degraded, acting [0,1] 2024-09-16T09:44:59.635 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.19 is active+undersized+degraded, acting [1,4] 2024-09-16T09:44:59.635 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1e is active+undersized+degraded, acting [6,2] 2024-09-16T09:44:59.635 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.0 is active+undersized+degraded, acting [7,0] 2024-09-16T09:44:59.635 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.14 is active+undersized+degraded, acting [1,7] 2024-09-16T09:44:59.635 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.19 is active+undersized+degraded, acting [2,0] 2024-09-16T09:44:59.930 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:59 smithi078 ceph-mon[63909]: from='client.34314 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:44:59.930 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:59 smithi078 ceph-mon[63909]: pgmap v173: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.8 MiB/s wr, 510 op/s; 204/1626 objects degraded (12.546%) 2024-09-16T09:44:59.930 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:59 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 204/1626 objects degraded (12.546%), 27 pgs degraded (PG_DEGRADED) 2024-09-16T09:44:59.930 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:44:59 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/2308383705' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:45:00.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:59 smithi088 ceph-mon[52758]: from='client.34314 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:45:00.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:59 smithi088 ceph-mon[52758]: pgmap v173: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.8 MiB/s wr, 510 op/s; 204/1626 objects degraded (12.546%) 2024-09-16T09:45:00.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:59 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 204/1626 objects degraded (12.546%), 27 pgs degraded (PG_DEGRADED) 2024-09-16T09:45:00.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:44:59 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/2308383705' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:45:00.569 INFO:teuthology.orchestra.run.smithi078.stdout: 5 214553 9.81 MB/sec execute 440 sec latency 43.863 ms 2024-09-16T09:45:00.966 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:00 smithi078 ceph-mon[63909]: from='client.34322 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:45:00.967 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:00 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:00.967 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:00 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:01.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:00 smithi088 ceph-mon[52758]: from='client.34322 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:45:01.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:00 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:01.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:00 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:01.569 INFO:teuthology.orchestra.run.smithi078.stdout: 5 214916 9.81 MB/sec execute 441 sec latency 58.952 ms 2024-09-16T09:45:01.729 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:45:01 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3[75896]: 2024-09-16T09:45:01.505+0000 7f00f81df640 -1 osd.3 0 read_superblock omap replica is missing. 2024-09-16T09:45:01.774 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:01 smithi078 ceph-mon[63909]: pgmap v174: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.7 MiB/s wr, 426 op/s; 207/1614 objects degraded (12.825%) 2024-09-16T09:45:01.774 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:01 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:01.774 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:01 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:02.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:01 smithi088 ceph-mon[52758]: pgmap v174: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.7 MiB/s wr, 426 op/s; 207/1614 objects degraded (12.825%) 2024-09-16T09:45:02.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:01 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:02.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:01 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:02.569 INFO:teuthology.orchestra.run.smithi078.stdout: 5 215216 9.82 MB/sec execute 442 sec latency 112.075 ms 2024-09-16T09:45:03.012 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:45:02 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3[75896]: 2024-09-16T09:45:02.768+0000 7f00f81df640 -1 osd.3 112 log_to_monitors true 2024-09-16T09:45:03.569 INFO:teuthology.orchestra.run.smithi078.stdout: 5 215592 9.81 MB/sec execute 443 sec latency 144.141 ms 2024-09-16T09:45:03.773 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:03 smithi078 ceph-mon[63909]: pgmap v175: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.2 MiB/s wr, 407 op/s; 208/1626 objects degraded (12.792%) 2024-09-16T09:45:03.773 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:03 smithi078 ceph-mon[63909]: from='osd.3 [v2:172.21.15.78:6826/2163039765,v1:172.21.15.78:6827/2163039765]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-16T09:45:04.039 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:45:03 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3[75896]: 2024-09-16T09:45:03.803+0000 7f00eff89640 -1 osd.3 112 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T09:45:04.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:03 smithi088 ceph-mon[52758]: pgmap v175: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.2 MiB/s wr, 407 op/s; 208/1626 objects degraded (12.792%) 2024-09-16T09:45:04.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:03 smithi088 ceph-mon[52758]: from='osd.3 [v2:172.21.15.78:6826/2163039765,v1:172.21.15.78:6827/2163039765]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-16T09:45:04.569 INFO:teuthology.orchestra.run.smithi078.stdout: 5 215999 9.80 MB/sec execute 444 sec latency 46.418 ms 2024-09-16T09:45:04.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:04 smithi078 ceph-mon[63909]: from='osd.3 [v2:172.21.15.78:6826/2163039765,v1:172.21.15.78:6827/2163039765]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-16T09:45:04.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:04 smithi078 ceph-mon[63909]: osdmap e115: 8 total, 7 up, 8 in 2024-09-16T09:45:04.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:04 smithi078 ceph-mon[63909]: from='osd.3 [v2:172.21.15.78:6826/2163039765,v1:172.21.15.78:6827/2163039765]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]} : dispatch 2024-09-16T09:45:04.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:04 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 208/1626 objects degraded (12.792%), 27 pgs degraded (PG_DEGRADED) 2024-09-16T09:45:05.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:04 smithi088 ceph-mon[52758]: from='osd.3 [v2:172.21.15.78:6826/2163039765,v1:172.21.15.78:6827/2163039765]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-16T09:45:05.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:04 smithi088 ceph-mon[52758]: osdmap e115: 8 total, 7 up, 8 in 2024-09-16T09:45:05.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:04 smithi088 ceph-mon[52758]: from='osd.3 [v2:172.21.15.78:6826/2163039765,v1:172.21.15.78:6827/2163039765]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]} : dispatch 2024-09-16T09:45:05.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:04 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 208/1626 objects degraded (12.792%), 27 pgs degraded (PG_DEGRADED) 2024-09-16T09:45:05.569 INFO:teuthology.orchestra.run.smithi078.stdout: 5 216092 9.79 MB/sec execute 445 sec latency 758.647 ms 2024-09-16T09:45:06.010 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:05 smithi078 ceph-mon[63909]: pgmap v177: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 489 op/s; 214/1611 objects degraded (13.284%) 2024-09-16T09:45:06.010 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:05 smithi078 ceph-mon[63909]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T09:45:06.010 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:05 smithi078 ceph-mon[63909]: osd.3 [v2:172.21.15.78:6826/2163039765,v1:172.21.15.78:6827/2163039765] boot 2024-09-16T09:45:06.010 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:05 smithi078 ceph-mon[63909]: osdmap e116: 8 total, 8 up, 8 in 2024-09-16T09:45:06.010 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:05 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-16T09:45:06.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:05 smithi088 ceph-mon[52758]: pgmap v177: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 489 op/s; 214/1611 objects degraded (13.284%) 2024-09-16T09:45:06.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:05 smithi088 ceph-mon[52758]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T09:45:06.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:05 smithi088 ceph-mon[52758]: osd.3 [v2:172.21.15.78:6826/2163039765,v1:172.21.15.78:6827/2163039765] boot 2024-09-16T09:45:06.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:05 smithi088 ceph-mon[52758]: osdmap e116: 8 total, 8 up, 8 in 2024-09-16T09:45:06.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:05 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-16T09:45:06.569 INFO:teuthology.orchestra.run.smithi078.stdout: 5 216111 9.77 MB/sec execute 446 sec latency 1758.671 ms 2024-09-16T09:45:06.937 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:06 smithi078 ceph-mon[63909]: osdmap e117: 8 total, 8 up, 8 in 2024-09-16T09:45:06.937 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:06.937 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:06.937 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:45:06.937 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:45:06.937 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:06.937 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:07.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:06 smithi088 ceph-mon[52758]: osdmap e117: 8 total, 8 up, 8 in 2024-09-16T09:45:07.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:07.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:07.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:45:07.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:45:07.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:07.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:07.569 INFO:teuthology.orchestra.run.smithi078.stdout: 5 216259 9.75 MB/sec execute 447 sec latency 2052.101 ms 2024-09-16T09:45:07.959 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:07 smithi078 ceph-mon[63909]: pgmap v180: 97 pgs: 6 peering, 12 active+undersized, 23 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.8 MiB/s wr, 401 op/s; 196/1611 objects degraded (12.166%) 2024-09-16T09:45:07.959 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:07 smithi078 ceph-mon[63909]: osdmap e118: 8 total, 8 up, 8 in 2024-09-16T09:45:07.960 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:07 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:45:07.960 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:07 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:45:07.960 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:07 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:45:07.960 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:07 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:45:07.960 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:07 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-16T09:45:08.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:07 smithi088 ceph-mon[52758]: pgmap v180: 97 pgs: 6 peering, 12 active+undersized, 23 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.8 MiB/s wr, 401 op/s; 196/1611 objects degraded (12.166%) 2024-09-16T09:45:08.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:07 smithi088 ceph-mon[52758]: osdmap e118: 8 total, 8 up, 8 in 2024-09-16T09:45:08.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:07 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:45:08.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:07 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:45:08.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:07 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:45:08.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:07 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:45:08.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:07 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-16T09:45:08.569 INFO:teuthology.orchestra.run.smithi078.stdout: 5 216617 9.76 MB/sec execute 448 sec latency 64.237 ms 2024-09-16T09:45:09.010 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:08 smithi078 ceph-mon[63909]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-16T09:45:09.010 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:08 smithi078 ceph-mon[63909]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-16T09:45:09.011 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:08 smithi078 ceph-mon[63909]: osdmap e119: 8 total, 8 up, 8 in 2024-09-16T09:45:09.300 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:08 smithi088 ceph-mon[52758]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-16T09:45:09.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:08 smithi088 ceph-mon[52758]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-16T09:45:09.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:08 smithi088 ceph-mon[52758]: osdmap e119: 8 total, 8 up, 8 in 2024-09-16T09:45:09.569 INFO:teuthology.orchestra.run.smithi078.stdout: 5 216763 9.74 MB/sec execute 449 sec latency 675.930 ms 2024-09-16T09:45:10.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:09 smithi078 ceph-mon[63909]: pgmap v183: 97 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 peering, 6 active+undersized, 16 active+undersized+degraded, 67 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 1.1 MiB/s wr, 71 op/s; 163/1620 objects degraded (10.062%); 29 KiB/s, 9 objects/s recovering 2024-09-16T09:45:10.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:09 smithi078 ceph-mon[63909]: osdmap e120: 8 total, 8 up, 8 in 2024-09-16T09:45:10.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:09 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 163/1620 objects degraded (10.062%), 17 pgs degraded (PG_DEGRADED) 2024-09-16T09:45:10.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:09 smithi088 ceph-mon[52758]: pgmap v183: 97 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 peering, 6 active+undersized, 16 active+undersized+degraded, 67 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 1.1 MiB/s wr, 71 op/s; 163/1620 objects degraded (10.062%); 29 KiB/s, 9 objects/s recovering 2024-09-16T09:45:10.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:09 smithi088 ceph-mon[52758]: osdmap e120: 8 total, 8 up, 8 in 2024-09-16T09:45:10.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:09 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 163/1620 objects degraded (10.062%), 17 pgs degraded (PG_DEGRADED) 2024-09-16T09:45:10.569 INFO:teuthology.orchestra.run.smithi078.stdout: 5 216903 9.73 MB/sec execute 450 sec latency 937.593 ms 2024-09-16T09:45:10.863 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:10 smithi078 ceph-mon[63909]: osdmap e121: 8 total, 8 up, 8 in 2024-09-16T09:45:10.863 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:10 smithi078 ceph-mon[63909]: osdmap e122: 8 total, 8 up, 8 in 2024-09-16T09:45:11.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:10 smithi088 ceph-mon[52758]: osdmap e121: 8 total, 8 up, 8 in 2024-09-16T09:45:11.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:10 smithi088 ceph-mon[52758]: osdmap e122: 8 total, 8 up, 8 in 2024-09-16T09:45:11.569 INFO:teuthology.orchestra.run.smithi078.stdout: 5 217101 9.72 MB/sec execute 451 sec latency 1017.116 ms 2024-09-16T09:45:11.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:11 smithi078 ceph-mon[63909]: pgmap v186: 97 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 7 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 81 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.1 MiB/s wr, 348 op/s; 138/1572 objects degraded (8.779%); 52 KiB/s, 21 objects/s recovering 2024-09-16T09:45:12.300 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:11 smithi088 ceph-mon[52758]: pgmap v186: 97 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 7 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 81 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.1 MiB/s wr, 348 op/s; 138/1572 objects degraded (8.779%); 52 KiB/s, 21 objects/s recovering 2024-09-16T09:45:12.569 INFO:teuthology.orchestra.run.smithi078.stdout: 5 217402 9.74 MB/sec execute 452 sec latency 51.509 ms 2024-09-16T09:45:13.426 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:13 smithi078 ceph-mon[63909]: pgmap v188: 97 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 87 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.2 MiB/s wr, 341 op/s; 138/1554 objects degraded (8.880%); 52 KiB/s, 26 objects/s recovering 2024-09-16T09:45:13.427 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:13 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:13.427 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:13 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:45:13.569 INFO:teuthology.orchestra.run.smithi078.stdout: 5 217647 9.73 MB/sec execute 453 sec latency 61.881 ms 2024-09-16T09:45:13.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:13 smithi088 ceph-mon[52758]: pgmap v188: 97 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 87 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.2 MiB/s wr, 341 op/s; 138/1554 objects degraded (8.880%); 52 KiB/s, 26 objects/s recovering 2024-09-16T09:45:13.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:13 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:13.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:13 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:45:14.431 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:14 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 138/1554 objects degraded (8.880%), 8 pgs degraded (PG_DEGRADED) 2024-09-16T09:45:14.569 INFO:teuthology.orchestra.run.smithi078.stdout: 5 217949 9.72 MB/sec execute 454 sec latency 74.160 ms 2024-09-16T09:45:14.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:14 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 138/1554 objects degraded (8.880%), 8 pgs degraded (PG_DEGRADED) 2024-09-16T09:45:15.482 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:15 smithi078 ceph-mon[63909]: pgmap v189: 97 pgs: 97 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.8 KiB/s rd, 4.9 MiB/s wr, 467 op/s; 948 KiB/s, 41 objects/s recovering 2024-09-16T09:45:15.482 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:15 smithi078 ceph-mon[63909]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 138/1554 objects degraded (8.880%), 8 pgs degraded) 2024-09-16T09:45:15.483 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:15 smithi078 ceph-mon[63909]: Cluster is now healthy 2024-09-16T09:45:15.569 INFO:teuthology.orchestra.run.smithi078.stdout: 5 218188 9.70 MB/sec execute 455 sec latency 52.000 ms 2024-09-16T09:45:15.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:15 smithi088 ceph-mon[52758]: pgmap v189: 97 pgs: 97 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.8 KiB/s rd, 4.9 MiB/s wr, 467 op/s; 948 KiB/s, 41 objects/s recovering 2024-09-16T09:45:15.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:15 smithi088 ceph-mon[52758]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 138/1554 objects degraded (8.880%), 8 pgs degraded) 2024-09-16T09:45:15.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:15 smithi088 ceph-mon[52758]: Cluster is now healthy 2024-09-16T09:45:16.569 INFO:teuthology.orchestra.run.smithi078.stdout: 5 218412 9.69 MB/sec execute 456 sec latency 80.560 ms 2024-09-16T09:45:17.555 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:17 smithi078 ceph-mon[63909]: pgmap v190: 97 pgs: 97 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 3.4 MiB/s wr, 249 op/s; 790 KiB/s, 30 objects/s recovering 2024-09-16T09:45:17.577 INFO:teuthology.orchestra.run.smithi078.stdout: 5 218695 9.69 MB/sec execute 457 sec latency 71.085 ms 2024-09-16T09:45:17.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:17 smithi088 ceph-mon[52758]: pgmap v190: 97 pgs: 97 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 3.4 MiB/s wr, 249 op/s; 790 KiB/s, 30 objects/s recovering 2024-09-16T09:45:18.577 INFO:teuthology.orchestra.run.smithi078.stdout: 5 218955 9.69 MB/sec execute 458 sec latency 83.393 ms 2024-09-16T09:45:19.393 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:19 smithi078 ceph-mon[63909]: pgmap v191: 97 pgs: 97 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.1 MiB/s wr, 261 op/s; 683 KiB/s, 24 objects/s recovering 2024-09-16T09:45:19.577 INFO:teuthology.orchestra.run.smithi078.stdout: 5 219248 9.68 MB/sec execute 459 sec latency 76.151 ms 2024-09-16T09:45:19.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:19 smithi088 ceph-mon[52758]: pgmap v191: 97 pgs: 97 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.1 MiB/s wr, 261 op/s; 683 KiB/s, 24 objects/s recovering 2024-09-16T09:45:20.577 INFO:teuthology.orchestra.run.smithi078.stdout: 5 219533 9.67 MB/sec execute 460 sec latency 47.798 ms 2024-09-16T09:45:21.430 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:21 smithi078 ceph-mon[63909]: pgmap v192: 97 pgs: 97 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.9 MiB/s wr, 368 op/s; 559 KiB/s, 20 objects/s recovering 2024-09-16T09:45:21.577 INFO:teuthology.orchestra.run.smithi078.stdout: 5 219727 9.66 MB/sec execute 461 sec latency 83.490 ms 2024-09-16T09:45:21.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:21 smithi088 ceph-mon[52758]: pgmap v192: 97 pgs: 97 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.9 MiB/s wr, 368 op/s; 559 KiB/s, 20 objects/s recovering 2024-09-16T09:45:22.449 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:22 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-16T09:45:22.577 INFO:teuthology.orchestra.run.smithi078.stdout: 5 220096 9.66 MB/sec execute 462 sec latency 47.403 ms 2024-09-16T09:45:22.594 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:22 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-16T09:45:23.467 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:23 smithi078 ceph-mon[63909]: pgmap v193: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.5 MiB/s wr, 335 op/s; 489 KiB/s, 15 objects/s recovering 2024-09-16T09:45:23.468 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:23 smithi078 ceph-mon[63909]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-16T09:45:23.468 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:23 smithi078 ceph-mon[63909]: Upgrade: osd.4 is safe to restart 2024-09-16T09:45:23.468 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:23 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:23.577 INFO:teuthology.orchestra.run.smithi078.stdout: 5 220464 9.67 MB/sec execute 463 sec latency 49.316 ms 2024-09-16T09:45:23.795 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:23 smithi088 ceph-mon[52758]: pgmap v193: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.5 MiB/s wr, 335 op/s; 489 KiB/s, 15 objects/s recovering 2024-09-16T09:45:23.795 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:23 smithi088 ceph-mon[52758]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-16T09:45:23.795 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:23 smithi088 ceph-mon[52758]: Upgrade: osd.4 is safe to restart 2024-09-16T09:45:23.795 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:23 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:24.577 INFO:teuthology.orchestra.run.smithi078.stdout: 5 220918 9.68 MB/sec execute 464 sec latency 62.233 ms 2024-09-16T09:45:24.999 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:24 smithi078 ceph-mon[63909]: Upgrade: Updating osd.4 2024-09-16T09:45:24.999 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:24 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:25.000 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:24 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-16T09:45:25.000 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:24 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:45:25.000 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:24 smithi078 ceph-mon[63909]: Deploying daemon osd.4 on smithi088 2024-09-16T09:45:25.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:24 smithi088 ceph-mon[52758]: Upgrade: Updating osd.4 2024-09-16T09:45:25.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:24 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:25.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:24 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-16T09:45:25.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:24 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:45:25.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:24 smithi088 ceph-mon[52758]: Deploying daemon osd.4 on smithi088 2024-09-16T09:45:25.577 INFO:teuthology.orchestra.run.smithi078.stdout: 5 221247 9.69 MB/sec execute 465 sec latency 77.711 ms 2024-09-16T09:45:25.900 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:25 smithi088 ceph-mon[52758]: pgmap v194: 97 pgs: 97 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.6 MiB/s wr, 444 op/s; 456 KiB/s, 14 objects/s recovering 2024-09-16T09:45:26.015 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:25 smithi078 ceph-mon[63909]: pgmap v194: 97 pgs: 97 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.6 MiB/s wr, 444 op/s; 456 KiB/s, 14 objects/s recovering 2024-09-16T09:45:26.577 INFO:teuthology.orchestra.run.smithi078.stdout: 5 221441 9.67 MB/sec execute 466 sec latency 106.507 ms 2024-09-16T09:45:27.577 INFO:teuthology.orchestra.run.smithi078.stdout: 5 221690 9.66 MB/sec execute 467 sec latency 129.876 ms 2024-09-16T09:45:27.662 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:27 smithi088 systemd[1]: Stopping Ceph osd.4 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:45:28.050 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:27 smithi078 ceph-mon[63909]: pgmap v195: 97 pgs: 97 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.7 MiB/s wr, 417 op/s 2024-09-16T09:45:28.050 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:27 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:28.051 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:27 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:45:28.051 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:27 smithi078 ceph-mon[63909]: osd.4 marked itself down and dead 2024-09-16T09:45:28.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:27 smithi088 ceph-mon[52758]: pgmap v195: 97 pgs: 97 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.7 MiB/s wr, 417 op/s 2024-09-16T09:45:28.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:27 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:28.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:27 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:45:28.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:27 smithi088 ceph-mon[52758]: osd.4 marked itself down and dead 2024-09-16T09:45:28.052 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:27 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4[33375]: 2024-09-16T09:45:27.658+0000 7fbd067d0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T09:45:28.052 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:27 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4[33375]: 2024-09-16T09:45:27.658+0000 7fbd067d0700 -1 osd.4 122 *** Got signal Terminated *** 2024-09-16T09:45:28.052 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:27 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4[33375]: 2024-09-16T09:45:27.658+0000 7fbd067d0700 -1 osd.4 122 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T09:45:28.577 INFO:teuthology.orchestra.run.smithi078.stdout: 5 221816 9.64 MB/sec execute 468 sec latency 286.499 ms 2024-09-16T09:45:29.221 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:28 smithi088 ceph-mon[52758]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T09:45:29.221 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:28 smithi088 ceph-mon[52758]: osdmap e123: 8 total, 7 up, 8 in 2024-09-16T09:45:29.221 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:28 smithi088 podman[56181]: 2024-09-16 09:45:28.924993662 +0000 UTC m=+1.382780271 container died 6f4e80fe127a1dc529527b2f0f4887903df55ac60f9ecf3aa5e9e17e6d70edf6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , architecture=x86_64, vendor=Red Hat, Inc., release=754, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, name=centos-stream, ceph=True, version=8, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-16T09:45:29.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:28 smithi078 ceph-mon[63909]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T09:45:29.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:28 smithi078 ceph-mon[63909]: osdmap e123: 8 total, 7 up, 8 in 2024-09-16T09:45:29.551 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:29 smithi088 podman[56181]: 2024-09-16 09:45:29.31042328 +0000 UTC m=+1.768209897 container cleanup 6f4e80fe127a1dc529527b2f0f4887903df55ac60f9ecf3aa5e9e17e6d70edf6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4, io.buildah.version=1.19.8, ceph=True, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, architecture=x86_64, version=8, build-date=2022-05-03T08:36:31.336870, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754) 2024-09-16T09:45:29.551 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:29 smithi088 bash[56181]: ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4 2024-09-16T09:45:29.577 INFO:teuthology.orchestra.run.smithi078.stdout: 5 221859 9.62 MB/sec execute 469 sec latency 1056.490 ms 2024-09-16T09:45:29.870 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:29 smithi088 podman[56195]: 2024-09-16 09:45:29.587111671 +0000 UTC m=+0.657329674 container remove 6f4e80fe127a1dc529527b2f0f4887903df55ac60f9ecf3aa5e9e17e6d70edf6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4, release=754, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., RELEASE=HEAD, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, ceph=True, io.buildah.version=1.19.8, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, distribution-scope=public, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-16T09:45:30.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:29 smithi078 ceph-mon[63909]: pgmap v196: 97 pgs: 97 active+clean; 356 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.2 MiB/s wr, 487 op/s 2024-09-16T09:45:30.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:29 smithi078 ceph-mon[63909]: osdmap e124: 8 total, 7 up, 8 in 2024-09-16T09:45:30.129 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:29 smithi088 ceph-mon[52758]: pgmap v196: 97 pgs: 97 active+clean; 356 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.2 MiB/s wr, 487 op/s 2024-09-16T09:45:30.130 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:29 smithi088 ceph-mon[52758]: osdmap e124: 8 total, 7 up, 8 in 2024-09-16T09:45:30.130 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:29 smithi088 podman[56287]: 2024-09-16 09:45:29.861975917 +0000 UTC m=+0.016388604 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:45:30.130 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:29 smithi088 podman[56287]: 2024-09-16 09:45:29.996273505 +0000 UTC m=+0.150686180 container create b2fbf6dca63d37be2379d6bae36d065fc1347cd54539d881596a16de8001e820 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-16T09:45:30.289 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-16T09:45:30.483 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:30 smithi088 podman[56287]: 2024-09-16 09:45:30.265660223 +0000 UTC m=+0.420072897 container init b2fbf6dca63d37be2379d6bae36d065fc1347cd54539d881596a16de8001e820 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-16T09:45:30.483 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:30 smithi088 podman[56287]: 2024-09-16 09:45:30.27045541 +0000 UTC m=+0.424868088 container start b2fbf6dca63d37be2379d6bae36d065fc1347cd54539d881596a16de8001e820 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6) 2024-09-16T09:45:30.483 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:30 smithi088 podman[56287]: 2024-09-16 09:45:30.374111747 +0000 UTC m=+0.528524435 container attach b2fbf6dca63d37be2379d6bae36d065fc1347cd54539d881596a16de8001e820 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_BRANCH=HEAD) 2024-09-16T09:45:30.577 INFO:teuthology.orchestra.run.smithi078.stdout: 5 222189 9.62 MB/sec execute 470 sec latency 66.984 ms 2024-09-16T09:45:30.796 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:30 smithi088 podman[56311]: 2024-09-16 09:45:30.51123858 +0000 UTC m=+0.018529669 container died b2fbf6dca63d37be2379d6bae36d065fc1347cd54539d881596a16de8001e820 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-16T09:45:31.405 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T09:45:31.405 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (6m) 31s ago 17m 16.2M - 0.27.0 11f11916f8cd 3ebb6832fbcc 2024-09-16T09:45:31.405 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (3m) 31s ago 17m 6899k - 19.3.0-4882-g66e33d6a 261f2dc7b20f 72965213139c 2024-09-16T09:45:31.405 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi088 smithi088 running (3m) 3m ago 15m 6891k - 19.3.0-4882-g66e33d6a 261f2dc7b20f eac0e6855b1b 2024-09-16T09:45:31.406 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (6m) 31s ago 16m 45.2M - dad864ee21e9 4a91b97e29b1 2024-09-16T09:45:31.406 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.hiotnn smithi078 running (10m) 31s ago 10m 38.1M - 17.2.0 e1d6a67b021e c43db7767ab9 2024-09-16T09:45:31.406 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi088.dbunob smithi088 running (10m) 3m ago 10m 17.0M - 17.2.0 e1d6a67b021e d235f1bfbb05 2024-09-16T09:45:31.406 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rrfxge smithi078 *:8443,9283,8765 running (6m) 31s ago 19m 519M - 19.3.0-4882-g66e33d6a 261f2dc7b20f 0d25f2f30f1e 2024-09-16T09:45:31.406 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi088.ncxfdz smithi088 *:8443,9283,8765 running (5m) 3m ago 15m 453M - 19.3.0-4882-g66e33d6a 261f2dc7b20f e2d03f701acd 2024-09-16T09:45:31.406 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (4m) 31s ago 19m 58.2M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f f5a3a0fec757 2024-09-16T09:45:31.406 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi088 smithi088 running (4m) 3m ago 15m 37.7M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f 2e552147327a 2024-09-16T09:45:31.406 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.wawosh smithi078 *:2049 running (10m) 31s ago 10m 280M - 4.0 e1d6a67b021e 2544ed9d67f5 2024-09-16T09:45:31.406 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi088.bnccmw smithi088 *:2049 running (9m) 3m ago 9m 81.6M - 4.0 e1d6a67b021e e606adb67049 2024-09-16T09:45:31.406 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (6m) 31s ago 16m 9.82M - 1.7.0 72c9c2088986 895ac3ccb2f3 2024-09-16T09:45:31.406 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi088 smithi088 *:9100 running (6m) 3m ago 15m 10.9M - 1.7.0 72c9c2088986 c84a2e50de53 2024-09-16T09:45:31.406 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (3m) 31s ago 14m 348M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 85008b499cb7 2024-09-16T09:45:31.406 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (2m) 31s ago 14m 271M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 12c137fcba70 2024-09-16T09:45:31.406 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (88s) 31s ago 13m 211M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 23a8b69c5c64 2024-09-16T09:45:31.407 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (40s) 31s ago 13m 11.2M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 00c9aa84d41d 2024-09-16T09:45:31.407 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi088 running (13m) 3m ago 13m 505M 2704M 17.2.0 e1d6a67b021e 6f4e80fe127a 2024-09-16T09:45:31.407 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi088 running (12m) 3m ago 12m 322M 2704M 17.2.0 e1d6a67b021e 73e51edb7756 2024-09-16T09:45:31.407 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi088 running (12m) 3m ago 12m 503M 2704M 17.2.0 e1d6a67b021e 7c1459b1a2d9 2024-09-16T09:45:31.407 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi088 running (11m) 3m ago 11m 398M 2704M 17.2.0 e1d6a67b021e 37f619c2ddf9 2024-09-16T09:45:31.407 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (5m) 31s ago 15m 53.1M - 2.51.0 1d3b7f56885b 4c956ead9125 2024-09-16T09:45:31.551 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:31 smithi088 podman[56311]: 2024-09-16 09:45:31.112369311 +0000 UTC m=+0.619660431 container remove b2fbf6dca63d37be2379d6bae36d065fc1347cd54539d881596a16de8001e820 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-16T09:45:31.551 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:31 smithi088 systemd[1]: ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.4.service: Deactivated successfully. 2024-09-16T09:45:31.551 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:31 smithi088 systemd[1]: Stopped Ceph osd.4 for a70c765e-740d-11ef-bceb-c7b262605968. 2024-09-16T09:45:31.551 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:31 smithi088 systemd[1]: ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.4.service: Consumed 3min 47.780s CPU time. 2024-09-16T09:45:31.551 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:31 smithi088 systemd[1]: Starting Ceph osd.4 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:45:31.577 INFO:teuthology.orchestra.run.smithi078.stdout: 5 222539 9.62 MB/sec execute 471 sec latency 57.088 ms 2024-09-16T09:45:31.876 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:31 smithi088 podman[56407]: 2024-09-16 09:45:31.467914899 +0000 UTC m=+0.016765821 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:45:31.876 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:31 smithi088 podman[56407]: 2024-09-16 09:45:31.605324958 +0000 UTC m=+0.154175865 container create 90d38781636032b697b7518de862edfd53b82c6e4597c751b73e91c0f09709f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4-activate, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6) 2024-09-16T09:45:31.876 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:31 smithi088 podman[56407]: 2024-09-16 09:45:31.875083654 +0000 UTC m=+0.423934570 container init 90d38781636032b697b7518de862edfd53b82c6e4597c751b73e91c0f09709f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4-activate, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0) 2024-09-16T09:45:31.885 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:31 smithi078 ceph-mon[63909]: pgmap v199: 97 pgs: 13 peering, 9 stale+active+clean, 75 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 5.1 MiB/s wr, 521 op/s 2024-09-16T09:45:31.885 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:31 smithi078 ceph-mon[63909]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:45:31.885 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:31 smithi078 ceph-mon[63909]: from='client.44275 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:45:32.106 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:45:32.106 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-16T09:45:32.106 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:45:32.106 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:45:32.107 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-16T09:45:32.107 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:45:32.107 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:45:32.107 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-16T09:45:32.107 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-16T09:45:32.107 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 4 2024-09-16T09:45:32.107 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:45:32.107 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-16T09:45:32.107 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:45:32.107 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:45:32.107 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-16T09:45:32.107 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-16T09:45:32.107 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 8 2024-09-16T09:45:32.107 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-16T09:45:32.108 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:45:32.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:31 smithi088 ceph-mon[52758]: pgmap v199: 97 pgs: 13 peering, 9 stale+active+clean, 75 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 5.1 MiB/s wr, 521 op/s 2024-09-16T09:45:32.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:31 smithi088 ceph-mon[52758]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:45:32.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:31 smithi088 ceph-mon[52758]: from='client.44275 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:45:32.301 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:31 smithi088 podman[56407]: 2024-09-16 09:45:31.879192188 +0000 UTC m=+0.428043105 container start 90d38781636032b697b7518de862edfd53b82c6e4597c751b73e91c0f09709f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4-activate, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-16T09:45:32.301 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:31 smithi088 podman[56407]: 2024-09-16 09:45:31.966787882 +0000 UTC m=+0.515638793 container attach 90d38781636032b697b7518de862edfd53b82c6e4597c751b73e91c0f09709f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_CLEAN=True) 2024-09-16T09:45:32.301 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:32 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4-activate[56422]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:45:32.301 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:32 smithi088 bash[56407]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:45:32.301 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:32 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4-activate[56422]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:45:32.302 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:32 smithi088 bash[56407]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:45:32.578 INFO:teuthology.orchestra.run.smithi078.stdout: 5 222901 9.62 MB/sec execute 472 sec latency 49.068 ms 2024-09-16T09:45:32.685 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:45:32.685 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f", 2024-09-16T09:45:32.685 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-16T09:45:32.685 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T09:45:32.685 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-16T09:45:32.685 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-09-16T09:45:32.685 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-16T09:45:32.685 INFO:teuthology.orchestra.run.smithi078.stdout: "mon" 2024-09-16T09:45:32.685 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-16T09:45:32.685 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "10/23 daemons upgraded", 2024-09-16T09:45:32.686 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-09-16T09:45:32.686 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-16T09:45:32.686 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:45:33.173 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:32 smithi078 ceph-mon[63909]: from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:45:33.174 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:32 smithi078 ceph-mon[63909]: pgmap v200: 97 pgs: 4 active+undersized, 13 peering, 5 stale+active+clean, 5 active+undersized+degraded, 70 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 3.4 MiB/s wr, 365 op/s; 40/1623 objects degraded (2.465%) 2024-09-16T09:45:33.174 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:32 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/838155027' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:45:33.174 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:32 smithi078 ceph-mon[63909]: from='client.44283 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:45:33.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:32 smithi088 ceph-mon[52758]: from='client.34340 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:45:33.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:32 smithi088 ceph-mon[52758]: pgmap v200: 97 pgs: 4 active+undersized, 13 peering, 5 stale+active+clean, 5 active+undersized+degraded, 70 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 3.4 MiB/s wr, 365 op/s; 40/1623 objects degraded (2.465%) 2024-09-16T09:45:33.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:32 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/838155027' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:45:33.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:32 smithi088 ceph-mon[52758]: from='client.44283 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:45:33.374 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 40/1623 objects degraded (2.465%), 5 pgs degraded 2024-09-16T09:45:33.374 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-16T09:45:33.374 INFO:teuthology.orchestra.run.smithi078.stdout: osd.4 (root=default,host=smithi088) is down 2024-09-16T09:45:33.374 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 40/1623 objects degraded (2.465%), 5 pgs degraded 2024-09-16T09:45:33.374 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.6 is active+undersized+degraded, acting [1,6] 2024-09-16T09:45:33.374 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.d is active+undersized+degraded, acting [1,3] 2024-09-16T09:45:33.375 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.9 is active+undersized+degraded, acting [2,7] 2024-09-16T09:45:33.375 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.19 is active+undersized+degraded, acting [1,3] 2024-09-16T09:45:33.375 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1a is active+undersized+degraded, acting [1,2] 2024-09-16T09:45:33.577 INFO:teuthology.orchestra.run.smithi078.stdout: 5 223181 9.61 MB/sec execute 473 sec latency 42.578 ms 2024-09-16T09:45:33.689 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:33 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4-activate[56422]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-16T09:45:33.689 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:33 smithi088 bash[56407]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-16T09:45:33.689 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:33 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4-activate[56422]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-16T09:45:33.689 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:33 smithi088 bash[56407]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-16T09:45:33.689 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:33 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4-activate[56422]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-16T09:45:33.689 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:33 smithi088 bash[56407]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-16T09:45:33.945 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:33 smithi078 ceph-mon[63909]: Health check failed: Degraded data redundancy: 40/1623 objects degraded (2.465%), 5 pgs degraded (PG_DEGRADED) 2024-09-16T09:45:33.945 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:33 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/3167968083' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:45:34.029 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:33 smithi088 ceph-mon[52758]: Health check failed: Degraded data redundancy: 40/1623 objects degraded (2.465%), 5 pgs degraded (PG_DEGRADED) 2024-09-16T09:45:34.029 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:33 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/3167968083' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:45:34.030 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:33 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4-activate[56422]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-16T09:45:34.030 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:33 smithi088 bash[56407]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-16T09:45:34.030 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:33 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4-activate[56422]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-16T09:45:34.030 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:33 smithi088 bash[56407]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-16T09:45:34.030 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:33 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4-activate[56422]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-16T09:45:34.030 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:33 smithi088 bash[56407]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-16T09:45:34.030 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:33 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4-activate[56422]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-16T09:45:34.030 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:33 smithi088 bash[56407]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-16T09:45:34.030 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:33 smithi088 podman[56407]: 2024-09-16 09:45:33.719817921 +0000 UTC m=+2.268668838 container died 90d38781636032b697b7518de862edfd53b82c6e4597c751b73e91c0f09709f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4-activate, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-16T09:45:34.578 INFO:teuthology.orchestra.run.smithi078.stdout: 5 223461 9.61 MB/sec execute 474 sec latency 38.783 ms 2024-09-16T09:45:34.606 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:34 smithi088 podman[56565]: 2024-09-16 09:45:34.35494241 +0000 UTC m=+0.628793121 container remove 90d38781636032b697b7518de862edfd53b82c6e4597c751b73e91c0f09709f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4-activate, GIT_BRANCH=HEAD, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, ceph=True) 2024-09-16T09:45:34.607 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:34 smithi088 podman[56621]: 2024-09-16 09:45:34.479702484 +0000 UTC m=+0.017906513 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:45:34.859 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:34 smithi088 podman[56621]: 2024-09-16 09:45:34.605898192 +0000 UTC m=+0.144102220 container create 1d92e66b75d99e2f2c7aed2103af346bb772e9e20db20f0fd6e9f7e521fec78b (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4, org.label-schema.schema-version=1.0, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-16T09:45:35.003 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:34 smithi078 ceph-mon[63909]: pgmap v201: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.0 MiB/s wr, 318 op/s; 250/1587 objects degraded (15.753%) 2024-09-16T09:45:35.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:34 smithi088 ceph-mon[52758]: pgmap v201: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.0 MiB/s wr, 318 op/s; 250/1587 objects degraded (15.753%) 2024-09-16T09:45:35.301 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:34 smithi088 podman[56621]: 2024-09-16 09:45:34.858726735 +0000 UTC m=+0.396930766 container init 1d92e66b75d99e2f2c7aed2103af346bb772e9e20db20f0fd6e9f7e521fec78b (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-16T09:45:35.301 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:34 smithi088 podman[56621]: 2024-09-16 09:45:34.862196539 +0000 UTC m=+0.400400560 container start 1d92e66b75d99e2f2c7aed2103af346bb772e9e20db20f0fd6e9f7e521fec78b (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-16T09:45:35.301 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:34 smithi088 ceph-osd[56639]: -- 172.21.15.88:0/1815040726 <== mon.1 v2:172.21.15.88:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5649e1b821e0 con 0x5649e0d1dc00 2024-09-16T09:45:35.302 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:34 smithi088 bash[56621]: 1d92e66b75d99e2f2c7aed2103af346bb772e9e20db20f0fd6e9f7e521fec78b 2024-09-16T09:45:35.302 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:34 smithi088 systemd[1]: Started Ceph osd.4 for a70c765e-740d-11ef-bceb-c7b262605968. 2024-09-16T09:45:35.578 INFO:teuthology.orchestra.run.smithi078.stdout: 5 223791 9.61 MB/sec execute 475 sec latency 69.568 ms 2024-09-16T09:45:36.053 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:36 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:36.053 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:36 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:36.054 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:36 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:45:36.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:36 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:36.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:36 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:36.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:36 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:45:36.578 INFO:teuthology.orchestra.run.smithi078.stdout: 5 224038 9.61 MB/sec execute 476 sec latency 85.185 ms 2024-09-16T09:45:36.800 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:36 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4[56635]: 2024-09-16T09:45:36.491+0000 7eff3ecca640 -1 Falling back to public interface 2024-09-16T09:45:37.258 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:37 smithi078 ceph-mon[63909]: pgmap v202: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.7 MiB/s wr, 337 op/s; 252/1602 objects degraded (15.730%) 2024-09-16T09:45:37.300 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:37 smithi088 ceph-mon[52758]: pgmap v202: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.7 MiB/s wr, 337 op/s; 252/1602 objects degraded (15.730%) 2024-09-16T09:45:37.578 INFO:teuthology.orchestra.run.smithi078.stdout: 5 224307 9.61 MB/sec execute 477 sec latency 79.293 ms 2024-09-16T09:45:38.285 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:38 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 249/1599 objects degraded (15.572%), 28 pgs degraded (PG_DEGRADED) 2024-09-16T09:45:38.434 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:38 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 249/1599 objects degraded (15.572%), 28 pgs degraded (PG_DEGRADED) 2024-09-16T09:45:38.578 INFO:teuthology.orchestra.run.smithi078.stdout: 5 224497 9.60 MB/sec execute 478 sec latency 48.829 ms 2024-09-16T09:45:39.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:39 smithi078 ceph-mon[63909]: pgmap v203: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.5 MiB/s wr, 389 op/s; 249/1599 objects degraded (15.572%) 2024-09-16T09:45:39.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:39 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:39.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:39 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:39.550 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:39 smithi088 ceph-mon[52758]: pgmap v203: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.5 MiB/s wr, 389 op/s; 249/1599 objects degraded (15.572%) 2024-09-16T09:45:39.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:39 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:39.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:39 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:39.578 INFO:teuthology.orchestra.run.smithi078.stdout: 5 224759 9.60 MB/sec execute 479 sec latency 90.689 ms 2024-09-16T09:45:40.578 INFO:teuthology.orchestra.run.smithi078.stdout: 5 225000 9.59 MB/sec execute 480 sec latency 58.500 ms 2024-09-16T09:45:40.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:40 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:40.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:40 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:41.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:40 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:41.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:40 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:41.578 INFO:teuthology.orchestra.run.smithi078.stdout: 5 225235 9.57 MB/sec execute 481 sec latency 60.959 ms 2024-09-16T09:45:41.916 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:41 smithi078 ceph-mon[63909]: pgmap v204: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 381 op/s; 254/1617 objects degraded (15.708%) 2024-09-16T09:45:42.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:41 smithi088 ceph-mon[52758]: pgmap v204: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 381 op/s; 254/1617 objects degraded (15.708%) 2024-09-16T09:45:42.578 INFO:teuthology.orchestra.run.smithi078.stdout: 5 225413 9.56 MB/sec execute 482 sec latency 51.718 ms 2024-09-16T09:45:43.467 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:43 smithi078 ceph-mon[63909]: pgmap v205: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 3.7 MiB/s wr, 350 op/s; 254/1608 objects degraded (15.796%) 2024-09-16T09:45:43.467 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:43 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:43.467 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:43 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:45:43.467 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:43 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:43.467 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:43 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:43.467 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:43 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:45:43.467 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:43 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:45:43.467 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:43 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:43.467 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:43 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:43.551 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:43 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4[56635]: 2024-09-16T09:45:43.217+0000 7eff3ecca640 -1 osd.4 0 read_superblock omap replica is missing. 2024-09-16T09:45:43.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:43 smithi088 ceph-mon[52758]: pgmap v205: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 3.7 MiB/s wr, 350 op/s; 254/1608 objects degraded (15.796%) 2024-09-16T09:45:43.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:43 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:43.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:43 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:45:43.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:43 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:43.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:43 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:43.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:43 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:45:43.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:43 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:45:43.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:43 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:43.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:43 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:43.578 INFO:teuthology.orchestra.run.smithi078.stdout: 5 225716 9.56 MB/sec execute 483 sec latency 83.115 ms 2024-09-16T09:45:44.525 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:44 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:45:44.525 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:44 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:45:44.525 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:44 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:45:44.525 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:44 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:45:44.525 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:44 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-16T09:45:44.525 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:44 smithi078 ceph-mon[63909]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-16T09:45:44.525 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:44 smithi078 ceph-mon[63909]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-16T09:45:44.525 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:44 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 254/1608 objects degraded (15.796%), 28 pgs degraded (PG_DEGRADED) 2024-09-16T09:45:44.578 INFO:teuthology.orchestra.run.smithi078.stdout: 5 226052 9.56 MB/sec execute 484 sec latency 57.742 ms 2024-09-16T09:45:44.801 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:44 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4[56635]: 2024-09-16T09:45:44.346+0000 7eff3ecca640 -1 osd.4 122 log_to_monitors true 2024-09-16T09:45:44.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:44 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:45:44.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:44 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:45:44.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:44 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:45:44.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:44 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:45:44.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:44 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-16T09:45:44.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:44 smithi088 ceph-mon[52758]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-16T09:45:44.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:44 smithi088 ceph-mon[52758]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-16T09:45:44.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:44 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 254/1608 objects degraded (15.796%), 28 pgs degraded (PG_DEGRADED) 2024-09-16T09:45:45.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:45 smithi078 ceph-mon[63909]: pgmap v206: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.5 MiB/s wr, 430 op/s; 243/1566 objects degraded (15.517%) 2024-09-16T09:45:45.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:45 smithi078 ceph-mon[63909]: from='osd.4 [v2:172.21.15.88:6800/2265866644,v1:172.21.15.88:6801/2265866644]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-16T09:45:45.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:45 smithi078 ceph-mon[63909]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-16T09:45:45.578 INFO:teuthology.orchestra.run.smithi078.stdout: 5 226380 9.55 MB/sec execute 485 sec latency 59.887 ms 2024-09-16T09:45:45.801 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:45:45 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4[56635]: 2024-09-16T09:45:45.362+0000 7eff36a74640 -1 osd.4 122 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T09:45:45.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:45 smithi088 ceph-mon[52758]: pgmap v206: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.5 MiB/s wr, 430 op/s; 243/1566 objects degraded (15.517%) 2024-09-16T09:45:45.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:45 smithi088 ceph-mon[52758]: from='osd.4 [v2:172.21.15.88:6800/2265866644,v1:172.21.15.88:6801/2265866644]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-16T09:45:45.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:45 smithi088 ceph-mon[52758]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-16T09:45:46.578 INFO:teuthology.orchestra.run.smithi078.stdout: 5 226612 9.54 MB/sec execute 486 sec latency 236.494 ms 2024-09-16T09:45:46.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:46 smithi088 ceph-mon[52758]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-16T09:45:46.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:46 smithi088 ceph-mon[52758]: from='osd.4 [v2:172.21.15.88:6800/2265866644,v1:172.21.15.88:6801/2265866644]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi088", "root=default"]} : dispatch 2024-09-16T09:45:46.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:46 smithi088 ceph-mon[52758]: osdmap e125: 8 total, 7 up, 8 in 2024-09-16T09:45:46.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:46 smithi088 ceph-mon[52758]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi088", "root=default"]} : dispatch 2024-09-16T09:45:46.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:46 smithi078 ceph-mon[63909]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-16T09:45:46.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:46 smithi078 ceph-mon[63909]: from='osd.4 [v2:172.21.15.88:6800/2265866644,v1:172.21.15.88:6801/2265866644]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi088", "root=default"]} : dispatch 2024-09-16T09:45:46.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:46 smithi078 ceph-mon[63909]: osdmap e125: 8 total, 7 up, 8 in 2024-09-16T09:45:46.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:46 smithi078 ceph-mon[63909]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi088", "root=default"]} : dispatch 2024-09-16T09:45:47.578 INFO:teuthology.orchestra.run.smithi078.stdout: 5 226631 9.52 MB/sec execute 487 sec latency 1236.520 ms 2024-09-16T09:45:47.635 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:47 smithi078 ceph-mon[63909]: pgmap v208: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 336 op/s; 242/1557 objects degraded (15.543%) 2024-09-16T09:45:47.636 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:47 smithi078 ceph-mon[63909]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T09:45:47.636 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:47 smithi078 ceph-mon[63909]: osd.4 [v2:172.21.15.88:6800/2265866644,v1:172.21.15.88:6801/2265866644] boot 2024-09-16T09:45:47.636 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:47 smithi078 ceph-mon[63909]: osdmap e126: 8 total, 8 up, 8 in 2024-09-16T09:45:47.636 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:47 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-16T09:45:47.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:47 smithi088 ceph-mon[52758]: pgmap v208: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 336 op/s; 242/1557 objects degraded (15.543%) 2024-09-16T09:45:47.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:47 smithi088 ceph-mon[52758]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T09:45:47.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:47 smithi088 ceph-mon[52758]: osd.4 [v2:172.21.15.88:6800/2265866644,v1:172.21.15.88:6801/2265866644] boot 2024-09-16T09:45:47.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:47 smithi088 ceph-mon[52758]: osdmap e126: 8 total, 8 up, 8 in 2024-09-16T09:45:47.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:47 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-16T09:45:48.422 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:48 smithi078 ceph-mon[63909]: osdmap e127: 8 total, 8 up, 8 in 2024-09-16T09:45:48.422 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:48 smithi078 ceph-mon[63909]: osdmap e128: 8 total, 8 up, 8 in 2024-09-16T09:45:48.578 INFO:teuthology.orchestra.run.smithi078.stdout: 5 226762 9.51 MB/sec execute 488 sec latency 2112.827 ms 2024-09-16T09:45:48.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:48 smithi088 ceph-mon[52758]: osdmap e127: 8 total, 8 up, 8 in 2024-09-16T09:45:48.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:48 smithi088 ceph-mon[52758]: osdmap e128: 8 total, 8 up, 8 in 2024-09-16T09:45:49.439 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:49 smithi078 ceph-mon[63909]: pgmap v211: 97 pgs: 1 active+recovering+degraded, 1 remapped+peering, 2 active+recovery_wait+degraded, 13 active+undersized, 21 active+undersized+degraded, 59 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.4 MiB/s wr, 457 op/s; 227/1536 objects degraded (14.779%); 70 KiB/s, 2 objects/s recovering 2024-09-16T09:45:49.440 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:49 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 227/1536 objects degraded (14.779%), 24 pgs degraded (PG_DEGRADED) 2024-09-16T09:45:49.440 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:49 smithi078 ceph-mon[63909]: osdmap e129: 8 total, 8 up, 8 in 2024-09-16T09:45:49.578 INFO:teuthology.orchestra.run.smithi078.stdout: 5 227070 9.51 MB/sec execute 489 sec latency 90.488 ms 2024-09-16T09:45:49.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:49 smithi088 ceph-mon[52758]: pgmap v211: 97 pgs: 1 active+recovering+degraded, 1 remapped+peering, 2 active+recovery_wait+degraded, 13 active+undersized, 21 active+undersized+degraded, 59 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.4 MiB/s wr, 457 op/s; 227/1536 objects degraded (14.779%); 70 KiB/s, 2 objects/s recovering 2024-09-16T09:45:49.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:49 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 227/1536 objects degraded (14.779%), 24 pgs degraded (PG_DEGRADED) 2024-09-16T09:45:49.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:49 smithi088 ceph-mon[52758]: osdmap e129: 8 total, 8 up, 8 in 2024-09-16T09:45:50.578 INFO:teuthology.orchestra.run.smithi078.stdout: 5 227333 9.50 MB/sec execute 490 sec latency 361.832 ms 2024-09-16T09:45:51.283 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:51 smithi078 ceph-mon[63909]: pgmap v214: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded, 4 remapped+peering, 6 active+recovery_wait+degraded, 11 active+undersized, 8 active+undersized+degraded, 66 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 2.7 MiB/s wr, 402 op/s; 179/1488 objects degraded (12.030%); 200 KiB/s, 8 objects/s recovering 2024-09-16T09:45:51.283 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:51 smithi078 ceph-mon[63909]: osdmap e130: 8 total, 8 up, 8 in 2024-09-16T09:45:51.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:51 smithi088 ceph-mon[52758]: pgmap v214: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded, 4 remapped+peering, 6 active+recovery_wait+degraded, 11 active+undersized, 8 active+undersized+degraded, 66 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 2.7 MiB/s wr, 402 op/s; 179/1488 objects degraded (12.030%); 200 KiB/s, 8 objects/s recovering 2024-09-16T09:45:51.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:51 smithi088 ceph-mon[52758]: osdmap e130: 8 total, 8 up, 8 in 2024-09-16T09:45:51.578 INFO:teuthology.orchestra.run.smithi078.stdout: 5 227490 9.50 MB/sec execute 491 sec latency 808.449 ms 2024-09-16T09:45:52.303 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:52 smithi078 ceph-mon[63909]: osdmap e131: 8 total, 8 up, 8 in 2024-09-16T09:45:52.550 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:52 smithi088 ceph-mon[52758]: osdmap e131: 8 total, 8 up, 8 in 2024-09-16T09:45:52.578 INFO:teuthology.orchestra.run.smithi078.stdout: 5 227754 9.50 MB/sec execute 492 sec latency 1097.612 ms 2024-09-16T09:45:53.345 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:53 smithi078 ceph-mon[63909]: pgmap v217: 97 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded, 4 remapped+peering, 7 active+recovery_wait+degraded, 82 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.1 MiB/s wr, 312 op/s; 162/1521 objects degraded (10.651%); 380 KiB/s, 60 keys/s, 20 objects/s recovering 2024-09-16T09:45:53.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:53 smithi078 ceph-mon[63909]: osdmap e132: 8 total, 8 up, 8 in 2024-09-16T09:45:53.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:53 smithi088 ceph-mon[52758]: pgmap v217: 97 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded, 4 remapped+peering, 7 active+recovery_wait+degraded, 82 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.1 MiB/s wr, 312 op/s; 162/1521 objects degraded (10.651%); 380 KiB/s, 60 keys/s, 20 objects/s recovering 2024-09-16T09:45:53.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:53 smithi088 ceph-mon[52758]: osdmap e132: 8 total, 8 up, 8 in 2024-09-16T09:45:53.578 INFO:teuthology.orchestra.run.smithi078.stdout: 5 228048 9.51 MB/sec execute 493 sec latency 625.592 ms 2024-09-16T09:45:54.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:54 smithi078 ceph-mon[63909]: osdmap e133: 8 total, 8 up, 8 in 2024-09-16T09:45:54.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:54 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 162/1521 objects degraded (10.651%), 10 pgs degraded (PG_DEGRADED) 2024-09-16T09:45:54.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:54 smithi088 ceph-mon[52758]: osdmap e133: 8 total, 8 up, 8 in 2024-09-16T09:45:54.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:54 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 162/1521 objects degraded (10.651%), 10 pgs degraded (PG_DEGRADED) 2024-09-16T09:45:54.578 INFO:teuthology.orchestra.run.smithi078.stdout: 5 228333 9.50 MB/sec execute 494 sec latency 119.860 ms 2024-09-16T09:45:55.384 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:55 smithi078 ceph-mon[63909]: pgmap v220: 97 pgs: 3 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 92 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.9 MiB/s wr, 358 op/s; 41/1605 objects degraded (2.555%); 1.4 MiB/s, 60 keys/s, 61 objects/s recovering 2024-09-16T09:45:55.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:55 smithi088 ceph-mon[52758]: pgmap v220: 97 pgs: 3 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 92 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.9 MiB/s wr, 358 op/s; 41/1605 objects degraded (2.555%); 1.4 MiB/s, 60 keys/s, 61 objects/s recovering 2024-09-16T09:45:55.578 INFO:teuthology.orchestra.run.smithi078.stdout: 5 228639 9.49 MB/sec execute 495 sec latency 86.350 ms 2024-09-16T09:45:56.408 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:56 smithi078 ceph-mon[63909]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 41/1605 objects degraded (2.555%), 2 pgs degraded) 2024-09-16T09:45:56.408 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:56 smithi078 ceph-mon[63909]: Cluster is now healthy 2024-09-16T09:45:56.578 INFO:teuthology.orchestra.run.smithi078.stdout: 5 228796 9.48 MB/sec execute 496 sec latency 84.595 ms 2024-09-16T09:45:56.780 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:56 smithi088 ceph-mon[52758]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 41/1605 objects degraded (2.555%), 2 pgs degraded) 2024-09-16T09:45:56.780 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:56 smithi088 ceph-mon[52758]: Cluster is now healthy 2024-09-16T09:45:57.491 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:57 smithi078 ceph-mon[63909]: pgmap v221: 97 pgs: 2 peering, 95 active+clean; 350 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.8 MiB/s wr, 472 op/s; 1023 KiB/s, 41 keys/s, 49 objects/s recovering 2024-09-16T09:45:57.491 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:57 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:57.578 INFO:teuthology.orchestra.run.smithi078.stdout: 5 228985 9.47 MB/sec execute 497 sec latency 61.698 ms 2024-09-16T09:45:57.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:57 smithi088 ceph-mon[52758]: pgmap v221: 97 pgs: 2 peering, 95 active+clean; 350 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.8 MiB/s wr, 472 op/s; 1023 KiB/s, 41 keys/s, 49 objects/s recovering 2024-09-16T09:45:57.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:57 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:45:58.507 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:58 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:45:58.578 INFO:teuthology.orchestra.run.smithi078.stdout: 5 229161 9.46 MB/sec execute 498 sec latency 53.410 ms 2024-09-16T09:45:58.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:58 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:45:59.399 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:59 smithi088 ceph-mon[52758]: pgmap v222: 97 pgs: 2 peering, 95 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.8 MiB/s wr, 353 op/s; 706 KiB/s, 32 objects/s recovering 2024-09-16T09:45:59.399 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:59 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-16T09:45:59.399 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:59 smithi088 ceph-mon[52758]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-16T09:45:59.399 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:45:59 smithi088 ceph-mon[52758]: Upgrade: osd.5 is safe to restart 2024-09-16T09:45:59.520 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:59 smithi078 ceph-mon[63909]: pgmap v222: 97 pgs: 2 peering, 95 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.8 MiB/s wr, 353 op/s; 706 KiB/s, 32 objects/s recovering 2024-09-16T09:45:59.521 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:59 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-16T09:45:59.521 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:59 smithi078 ceph-mon[63909]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-16T09:45:59.521 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:45:59 smithi078 ceph-mon[63909]: Upgrade: osd.5 is safe to restart 2024-09-16T09:45:59.579 INFO:teuthology.orchestra.run.smithi078.stdout: 5 229454 9.46 MB/sec execute 499 sec latency 53.298 ms 2024-09-16T09:46:00.579 INFO:teuthology.orchestra.run.smithi078.stdout: 5 229725 9.45 MB/sec execute 500 sec latency 70.682 ms 2024-09-16T09:46:01.578 INFO:teuthology.orchestra.run.smithi078.stdout: 5 230053 9.44 MB/sec execute 501 sec latency 91.100 ms 2024-09-16T09:46:01.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:01 smithi078 ceph-mon[63909]: pgmap v223: 97 pgs: 97 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 4.2 MiB/s wr, 344 op/s; 602 KiB/s, 27 objects/s recovering 2024-09-16T09:46:01.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:01 smithi078 ceph-mon[63909]: Upgrade: Updating osd.5 2024-09-16T09:46:01.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:01 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:01.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:01 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-16T09:46:01.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:01 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:46:01.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:01 smithi078 ceph-mon[63909]: Deploying daemon osd.5 on smithi088 2024-09-16T09:46:02.022 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:01 smithi088 ceph-mon[52758]: pgmap v223: 97 pgs: 97 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 4.2 MiB/s wr, 344 op/s; 602 KiB/s, 27 objects/s recovering 2024-09-16T09:46:02.023 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:01 smithi088 ceph-mon[52758]: Upgrade: Updating osd.5 2024-09-16T09:46:02.023 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:01 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:02.023 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:01 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-16T09:46:02.023 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:01 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:46:02.023 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:01 smithi088 ceph-mon[52758]: Deploying daemon osd.5 on smithi088 2024-09-16T09:46:02.578 INFO:teuthology.orchestra.run.smithi078.stdout: 5 230237 9.43 MB/sec execute 502 sec latency 64.256 ms 2024-09-16T09:46:03.579 INFO:teuthology.orchestra.run.smithi078.stdout: 5 230437 9.42 MB/sec execute 503 sec latency 56.536 ms 2024-09-16T09:46:03.657 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:03 smithi078 ceph-mon[63909]: pgmap v224: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.7 MiB/s wr, 306 op/s; 491 KiB/s, 22 objects/s recovering 2024-09-16T09:46:03.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:03 smithi088 ceph-mon[52758]: pgmap v224: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.7 MiB/s wr, 306 op/s; 491 KiB/s, 22 objects/s recovering 2024-09-16T09:46:03.835 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-16T09:46:04.205 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:03 smithi088 systemd[1]: Stopping Ceph osd.5 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:46:04.551 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:04 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[36324]: 2024-09-16T09:46:04.203+0000 7f1552d37700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T09:46:04.551 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:04 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[36324]: 2024-09-16T09:46:04.203+0000 7f1552d37700 -1 osd.5 133 *** Got signal Terminated *** 2024-09-16T09:46:04.551 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:04 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[36324]: 2024-09-16T09:46:04.203+0000 7f1552d37700 -1 osd.5 133 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T09:46:04.579 INFO:teuthology.orchestra.run.smithi078.stdout: 5 230799 9.43 MB/sec execute 504 sec latency 56.405 ms 2024-09-16T09:46:04.680 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:04 smithi078 ceph-mon[63909]: from='client.34358 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:46:04.681 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:04 smithi078 ceph-mon[63909]: osd.5 marked itself down and dead 2024-09-16T09:46:04.785 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T09:46:04.785 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (7m) 64s ago 18m 16.2M - 0.27.0 11f11916f8cd 3ebb6832fbcc 2024-09-16T09:46:04.785 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (4m) 64s ago 18m 6899k - 19.3.0-4882-g66e33d6a 261f2dc7b20f 72965213139c 2024-09-16T09:46:04.785 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi088 smithi088 running (4m) 25s ago 16m 6891k - 19.3.0-4882-g66e33d6a 261f2dc7b20f eac0e6855b1b 2024-09-16T09:46:04.785 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (7m) 64s ago 16m 45.2M - dad864ee21e9 4a91b97e29b1 2024-09-16T09:46:04.785 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.hiotnn smithi078 running (10m) 64s ago 10m 38.1M - 17.2.0 e1d6a67b021e c43db7767ab9 2024-09-16T09:46:04.785 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi088.dbunob smithi088 running (10m) 25s ago 10m 17.9M - 17.2.0 e1d6a67b021e d235f1bfbb05 2024-09-16T09:46:04.786 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rrfxge smithi078 *:8443,9283,8765 running (6m) 64s ago 19m 519M - 19.3.0-4882-g66e33d6a 261f2dc7b20f 0d25f2f30f1e 2024-09-16T09:46:04.786 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi088.ncxfdz smithi088 *:8443,9283,8765 running (5m) 25s ago 16m 456M - 19.3.0-4882-g66e33d6a 261f2dc7b20f e2d03f701acd 2024-09-16T09:46:04.786 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (5m) 64s ago 19m 58.2M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f f5a3a0fec757 2024-09-16T09:46:04.786 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi088 smithi088 running (5m) 25s ago 16m 44.5M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f 2e552147327a 2024-09-16T09:46:04.786 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.wawosh smithi078 *:2049 running (10m) 64s ago 10m 280M - 4.0 e1d6a67b021e 2544ed9d67f5 2024-09-16T09:46:04.786 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi088.bnccmw smithi088 *:2049 running (10m) 25s ago 10m 82.1M - 4.0 e1d6a67b021e e606adb67049 2024-09-16T09:46:04.786 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (7m) 64s ago 16m 9.82M - 1.7.0 72c9c2088986 895ac3ccb2f3 2024-09-16T09:46:04.786 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi088 smithi088 *:9100 running (6m) 25s ago 16m 12.1M - 1.7.0 72c9c2088986 c84a2e50de53 2024-09-16T09:46:04.786 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (3m) 64s ago 15m 348M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 85008b499cb7 2024-09-16T09:46:04.786 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (2m) 64s ago 14m 271M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 12c137fcba70 2024-09-16T09:46:04.786 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (2m) 64s ago 14m 211M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 23a8b69c5c64 2024-09-16T09:46:04.786 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (73s) 64s ago 14m 11.2M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 00c9aa84d41d 2024-09-16T09:46:04.786 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi088 running (30s) 25s ago 13m 12.2M 2704M 19.3.0-4882-g66e33d6a 261f2dc7b20f 1d92e66b75d9 2024-09-16T09:46:04.787 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi088 running (13m) 25s ago 13m 530M 2704M 17.2.0 e1d6a67b021e 73e51edb7756 2024-09-16T09:46:04.787 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi088 running (12m) 25s ago 12m 452M 2704M 17.2.0 e1d6a67b021e 7c1459b1a2d9 2024-09-16T09:46:04.787 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi088 running (12m) 25s ago 12m 698M 2704M 17.2.0 e1d6a67b021e 37f619c2ddf9 2024-09-16T09:46:04.787 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (5m) 64s ago 15m 53.1M - 2.51.0 1d3b7f56885b 4c956ead9125 2024-09-16T09:46:05.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:04 smithi088 ceph-mon[52758]: from='client.34358 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:46:05.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:04 smithi088 ceph-mon[52758]: osd.5 marked itself down and dead 2024-09-16T09:46:05.243 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:46:05.243 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-16T09:46:05.243 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:46:05.243 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:46:05.244 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-16T09:46:05.244 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:46:05.244 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:46:05.244 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-16T09:46:05.244 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-16T09:46:05.244 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 5 2024-09-16T09:46:05.244 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:46:05.244 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-16T09:46:05.244 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:46:05.244 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:46:05.244 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-16T09:46:05.244 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-16T09:46:05.244 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 9 2024-09-16T09:46:05.245 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-16T09:46:05.245 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:46:05.486 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:05 smithi088 podman[58182]: 2024-09-16 09:46:05.162366621 +0000 UTC m=+1.078729821 container died 73e51edb7756add6e12cfa6ad3a40a767289e70cfca79ead55ed97801f09ee55 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, release=754, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, architecture=x86_64, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, distribution-scope=public, maintainer=Guillaume Abrioux , io.openshift.expose-services=, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870) 2024-09-16T09:46:05.487 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:05 smithi088 podman[58182]: 2024-09-16 09:46:05.485883839 +0000 UTC m=+1.402247040 container cleanup 73e51edb7756add6e12cfa6ad3a40a767289e70cfca79ead55ed97801f09ee55 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., version=8, ceph=True, com.redhat.component=centos-stream-container, name=centos-stream, GIT_CLEAN=True, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, vcs-type=git, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-16T09:46:05.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 230901 9.42 MB/sec execute 505 sec latency 892.869 ms 2024-09-16T09:46:05.598 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:46:05.598 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f", 2024-09-16T09:46:05.598 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-16T09:46:05.598 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T09:46:05.599 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-16T09:46:05.599 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-09-16T09:46:05.599 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-16T09:46:05.599 INFO:teuthology.orchestra.run.smithi078.stdout: "mon" 2024-09-16T09:46:05.599 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-16T09:46:05.599 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "11/23 daemons upgraded", 2024-09-16T09:46:05.599 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-09-16T09:46:05.599 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-16T09:46:05.599 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:46:05.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:05 smithi088 ceph-mon[52758]: pgmap v225: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.0 MiB/s wr, 416 op/s; 67 KiB/s, 8 objects/s recovering 2024-09-16T09:46:05.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:05 smithi088 ceph-mon[52758]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:46:05.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:05 smithi088 ceph-mon[52758]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T09:46:05.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:05 smithi088 ceph-mon[52758]: osdmap e134: 8 total, 7 up, 8 in 2024-09-16T09:46:05.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:05 smithi088 ceph-mon[52758]: from='client.44303 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:46:05.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:05 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/2960760479' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:46:05.802 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:05 smithi088 bash[58182]: ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5 2024-09-16T09:46:05.802 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:05 smithi088 podman[58201]: 2024-09-16 09:46:05.68739002 +0000 UTC m=+0.524265735 container remove 73e51edb7756add6e12cfa6ad3a40a767289e70cfca79ead55ed97801f09ee55 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , vcs-type=git, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-16T09:46:05.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:05 smithi078 ceph-mon[63909]: pgmap v225: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.0 MiB/s wr, 416 op/s; 67 KiB/s, 8 objects/s recovering 2024-09-16T09:46:05.850 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:05 smithi078 ceph-mon[63909]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:46:05.850 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:05 smithi078 ceph-mon[63909]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T09:46:05.850 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:05 smithi078 ceph-mon[63909]: osdmap e134: 8 total, 7 up, 8 in 2024-09-16T09:46:05.850 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:05 smithi078 ceph-mon[63909]: from='client.44303 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:46:05.850 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:05 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/2960760479' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:46:06.165 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_WARN 1 osds down 2024-09-16T09:46:06.166 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-16T09:46:06.166 INFO:teuthology.orchestra.run.smithi078.stdout: osd.5 (root=default,host=smithi088) is down 2024-09-16T09:46:06.450 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:06 smithi088 podman[58282]: 2024-09-16 09:46:06.074977951 +0000 UTC m=+0.017389255 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:46:06.451 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:06 smithi088 podman[58282]: 2024-09-16 09:46:06.205166453 +0000 UTC m=+0.147577752 container create 253eed49367d3184a13600194a3935036e64ce8c3f19289d003f2f2a918d1c6b (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-16T09:46:06.451 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:06 smithi088 podman[58282]: 2024-09-16 09:46:06.449715648 +0000 UTC m=+0.392126941 container init 253eed49367d3184a13600194a3935036e64ce8c3f19289d003f2f2a918d1c6b (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5-deactivate, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-16T09:46:06.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 231232 9.42 MB/sec execute 506 sec latency 975.102 ms 2024-09-16T09:46:06.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:06 smithi088 ceph-mon[52758]: from='client.34372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:46:06.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:06 smithi088 ceph-mon[52758]: osdmap e135: 8 total, 7 up, 8 in 2024-09-16T09:46:06.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:06 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/4290873479' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:46:06.801 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:06 smithi088 podman[58282]: 2024-09-16 09:46:06.454669251 +0000 UTC m=+0.397080542 container start 253eed49367d3184a13600194a3935036e64ce8c3f19289d003f2f2a918d1c6b (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-16T09:46:06.801 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:06 smithi088 podman[58282]: 2024-09-16 09:46:06.558047998 +0000 UTC m=+0.500459287 container attach 253eed49367d3184a13600194a3935036e64ce8c3f19289d003f2f2a918d1c6b (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5-deactivate, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-16T09:46:06.801 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:06 smithi088 podman[58282]: 2024-09-16 09:46:06.66694409 +0000 UTC m=+0.609355395 container died 253eed49367d3184a13600194a3935036e64ce8c3f19289d003f2f2a918d1c6b (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-16T09:46:06.873 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:06 smithi078 ceph-mon[63909]: from='client.34372 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:46:06.874 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:06 smithi078 ceph-mon[63909]: osdmap e135: 8 total, 7 up, 8 in 2024-09-16T09:46:06.874 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:06 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/4290873479' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:46:07.579 INFO:teuthology.orchestra.run.smithi078.stdout: 5 231502 9.43 MB/sec execute 507 sec latency 43.595 ms 2024-09-16T09:46:07.653 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:07 smithi088 podman[58305]: 2024-09-16 09:46:07.355143384 +0000 UTC m=+0.682071327 container remove 253eed49367d3184a13600194a3935036e64ce8c3f19289d003f2f2a918d1c6b (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-16T09:46:07.654 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:07 smithi088 systemd[1]: ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.5.service: Deactivated successfully. 2024-09-16T09:46:07.654 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:07 smithi088 systemd[1]: Stopped Ceph osd.5 for a70c765e-740d-11ef-bceb-c7b262605968. 2024-09-16T09:46:07.654 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:07 smithi088 systemd[1]: ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.5.service: Consumed 4min 39.657s CPU time. 2024-09-16T09:46:07.654 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:07 smithi088 systemd[1]: Starting Ceph osd.5 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:46:07.903 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:07 smithi078 ceph-mon[63909]: pgmap v228: 97 pgs: 10 peering, 9 stale+active+clean, 78 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.5 MiB/s wr, 315 op/s 2024-09-16T09:46:07.904 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:07 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:07.969 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:07 smithi088 ceph-mon[52758]: pgmap v228: 97 pgs: 10 peering, 9 stale+active+clean, 78 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.5 MiB/s wr, 315 op/s 2024-09-16T09:46:07.969 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:07 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:07.969 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:07 smithi088 podman[58402]: 2024-09-16 09:46:07.707184057 +0000 UTC m=+0.016468101 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:46:07.969 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:07 smithi088 podman[58402]: 2024-09-16 09:46:07.84011919 +0000 UTC m=+0.149403263 container create 91370dd16c21928fe796fc1b31934448a586485ba829b1d2cdb337146c164805 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-16T09:46:08.241 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:08 smithi088 podman[58402]: 2024-09-16 09:46:08.084731561 +0000 UTC m=+0.394015605 container init 91370dd16c21928fe796fc1b31934448a586485ba829b1d2cdb337146c164805 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-16T09:46:08.241 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:08 smithi088 podman[58402]: 2024-09-16 09:46:08.088198835 +0000 UTC m=+0.397482867 container start 91370dd16c21928fe796fc1b31934448a586485ba829b1d2cdb337146c164805 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5-activate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-16T09:46:08.241 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:08 smithi088 podman[58402]: 2024-09-16 09:46:08.176115455 +0000 UTC m=+0.485399509 container attach 91370dd16c21928fe796fc1b31934448a586485ba829b1d2cdb337146c164805 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-16T09:46:08.551 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:08 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5-activate[58417]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:46:08.551 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:08 smithi088 bash[58402]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:46:08.551 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:08 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5-activate[58417]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:46:08.551 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:08 smithi088 bash[58402]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:46:08.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 231717 9.42 MB/sec execute 508 sec latency 129.441 ms 2024-09-16T09:46:08.933 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:08 smithi078 ceph-mon[63909]: Health check failed: Degraded data redundancy: 29/1581 objects degraded (1.834%), 6 pgs degraded (PG_DEGRADED) 2024-09-16T09:46:09.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:08 smithi088 ceph-mon[52758]: Health check failed: Degraded data redundancy: 29/1581 objects degraded (1.834%), 6 pgs degraded (PG_DEGRADED) 2024-09-16T09:46:09.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 231952 9.41 MB/sec execute 509 sec latency 123.764 ms 2024-09-16T09:46:09.611 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:09 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5-activate[58417]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-16T09:46:09.611 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:09 smithi088 bash[58402]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-16T09:46:09.611 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:09 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5-activate[58417]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-16T09:46:09.611 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:09 smithi088 bash[58402]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-16T09:46:09.611 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:09 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5-activate[58417]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-16T09:46:09.612 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:09 smithi088 bash[58402]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-16T09:46:09.724 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:09 smithi078 ceph-mon[63909]: pgmap v229: 97 pgs: 2 active+undersized, 10 peering, 6 stale+active+clean, 6 active+undersized+degraded, 73 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 2.6 MiB/s wr, 275 op/s; 29/1581 objects degraded (1.834%) 2024-09-16T09:46:09.964 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:09 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5-activate[58417]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-16T09:46:09.964 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:09 smithi088 bash[58402]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-16T09:46:09.965 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:09 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5-activate[58417]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-16T09:46:09.965 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:09 smithi088 bash[58402]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-16T09:46:09.965 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:09 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5-activate[58417]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-16T09:46:09.965 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:09 smithi088 bash[58402]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-16T09:46:09.965 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:09 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5-activate[58417]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-16T09:46:09.965 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:09 smithi088 bash[58402]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-16T09:46:09.965 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:09 smithi088 podman[58402]: 2024-09-16 09:46:09.641390943 +0000 UTC m=+1.950674977 container died 91370dd16c21928fe796fc1b31934448a586485ba829b1d2cdb337146c164805 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_BRANCH=HEAD) 2024-09-16T09:46:09.966 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:09 smithi088 ceph-mon[52758]: pgmap v229: 97 pgs: 2 active+undersized, 10 peering, 6 stale+active+clean, 6 active+undersized+degraded, 73 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 2.6 MiB/s wr, 275 op/s; 29/1581 objects degraded (1.834%) 2024-09-16T09:46:10.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 232249 9.40 MB/sec execute 510 sec latency 64.422 ms 2024-09-16T09:46:10.732 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:10 smithi088 podman[58559]: 2024-09-16 09:46:10.389804981 +0000 UTC m=+0.740980608 container remove 91370dd16c21928fe796fc1b31934448a586485ba829b1d2cdb337146c164805 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-16T09:46:11.443 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:10 smithi088 podman[58616]: 2024-09-16 09:46:10.503504381 +0000 UTC m=+0.017580119 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:46:11.444 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:10 smithi088 podman[58616]: 2024-09-16 09:46:10.615978277 +0000 UTC m=+0.130054017 container create 43abc1197fe24d7d85901200df15501ecccff3a180319873191e8a650f78810d (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6) 2024-09-16T09:46:11.444 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:10 smithi088 podman[58616]: 2024-09-16 09:46:10.835518465 +0000 UTC m=+0.349594216 container init 43abc1197fe24d7d85901200df15501ecccff3a180319873191e8a650f78810d (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-16T09:46:11.444 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:10 smithi088 podman[58616]: 2024-09-16 09:46:10.839586327 +0000 UTC m=+0.353662074 container start 43abc1197fe24d7d85901200df15501ecccff3a180319873191e8a650f78810d (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-16T09:46:11.445 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:10 smithi088 ceph-osd[58635]: -- 172.21.15.88:0/4249339274 <== mon.1 v2:172.21.15.88:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e1264fc1e0 con 0x55e12569fc00 2024-09-16T09:46:11.445 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:10 smithi088 bash[58616]: 43abc1197fe24d7d85901200df15501ecccff3a180319873191e8a650f78810d 2024-09-16T09:46:11.445 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:10 smithi088 systemd[1]: Started Ceph osd.5 for a70c765e-740d-11ef-bceb-c7b262605968. 2024-09-16T09:46:11.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 232502 9.40 MB/sec execute 511 sec latency 95.277 ms 2024-09-16T09:46:11.796 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:11 smithi078 ceph-mon[63909]: pgmap v230: 97 pgs: 5 active+undersized, 10 peering, 17 active+undersized+degraded, 65 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 4.2 MiB/s wr, 439 op/s; 136/1617 objects degraded (8.411%) 2024-09-16T09:46:11.796 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:11 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:11.796 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:11 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:11.796 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:11 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:46:12.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:11 smithi088 ceph-mon[52758]: pgmap v230: 97 pgs: 5 active+undersized, 10 peering, 17 active+undersized+degraded, 65 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 4.2 MiB/s wr, 439 op/s; 136/1617 objects degraded (8.411%) 2024-09-16T09:46:12.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:11 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:12.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:11 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:12.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:11 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:46:12.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 232763 9.39 MB/sec execute 512 sec latency 53.960 ms 2024-09-16T09:46:12.845 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:12 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:46:12.716+0000 7fcc56755640 -1 Falling back to public interface 2024-09-16T09:46:13.334 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:13 smithi078 ceph-mon[63909]: pgmap v231: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.4 MiB/s wr, 304 op/s; 183/1662 objects degraded (11.011%) 2024-09-16T09:46:13.335 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:13 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:13.335 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:13 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:46:13.574 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:13 smithi088 ceph-mon[52758]: pgmap v231: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.4 MiB/s wr, 304 op/s; 183/1662 objects degraded (11.011%) 2024-09-16T09:46:13.574 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:13 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:13.574 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:13 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:46:13.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 233091 9.39 MB/sec execute 513 sec latency 54.501 ms 2024-09-16T09:46:14.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:14 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 183/1662 objects degraded (11.011%), 23 pgs degraded (PG_DEGRADED) 2024-09-16T09:46:14.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 233534 9.39 MB/sec execute 514 sec latency 38.475 ms 2024-09-16T09:46:14.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:14 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 183/1662 objects degraded (11.011%), 23 pgs degraded (PG_DEGRADED) 2024-09-16T09:46:15.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 233875 9.39 MB/sec execute 515 sec latency 49.182 ms 2024-09-16T09:46:15.869 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:15 smithi078 ceph-mon[63909]: pgmap v232: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 348 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.1 MiB/s wr, 363 op/s; 187/1620 objects degraded (11.543%) 2024-09-16T09:46:15.870 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:15 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:15.870 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:15 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:15.870 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:15 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:15.870 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:15 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:15.984 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:15 smithi088 ceph-mon[52758]: pgmap v232: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 348 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.1 MiB/s wr, 363 op/s; 187/1620 objects degraded (11.543%) 2024-09-16T09:46:15.985 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:15 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:15.985 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:15 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:15.985 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:15 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:15.985 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:15 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:16.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 234287 9.39 MB/sec execute 516 sec latency 60.062 ms 2024-09-16T09:46:17.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 234609 9.39 MB/sec execute 517 sec latency 79.485 ms 2024-09-16T09:46:17.935 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:17 smithi078 ceph-mon[63909]: pgmap v233: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.9 MiB/s wr, 399 op/s; 176/1560 objects degraded (11.282%) 2024-09-16T09:46:17.961 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:17 smithi088 ceph-mon[52758]: pgmap v233: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.9 MiB/s wr, 399 op/s; 176/1560 objects degraded (11.282%) 2024-09-16T09:46:18.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 234851 9.39 MB/sec execute 518 sec latency 51.200 ms 2024-09-16T09:46:19.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 235087 9.39 MB/sec execute 519 sec latency 55.753 ms 2024-09-16T09:46:19.758 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:19 smithi078 ceph-mon[63909]: pgmap v234: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.6 MiB/s wr, 403 op/s; 176/1560 objects degraded (11.282%) 2024-09-16T09:46:19.758 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:19.758 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:19.758 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:46:19.758 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:46:19.758 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:19.758 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:19.758 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:46:19.758 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:46:19.758 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:46:19.758 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:46:19.759 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-16T09:46:19.759 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:19 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 176/1560 objects degraded (11.282%), 23 pgs degraded (PG_DEGRADED) 2024-09-16T09:46:20.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:19 smithi088 ceph-mon[52758]: pgmap v234: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.6 MiB/s wr, 403 op/s; 176/1560 objects degraded (11.282%) 2024-09-16T09:46:20.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:20.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:20.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:46:20.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:46:20.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:20.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:20.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:46:20.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:46:20.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:46:20.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:46:20.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-16T09:46:20.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:19 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 176/1560 objects degraded (11.282%), 23 pgs degraded (PG_DEGRADED) 2024-09-16T09:46:20.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 235307 9.39 MB/sec execute 520 sec latency 53.652 ms 2024-09-16T09:46:20.802 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:20 smithi078 ceph-mon[63909]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-16T09:46:20.803 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:20 smithi078 ceph-mon[63909]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-16T09:46:21.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:20 smithi088 ceph-mon[52758]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-16T09:46:21.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:20 smithi088 ceph-mon[52758]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-16T09:46:21.051 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:20 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:46:20.673+0000 7fcc56755640 -1 osd.5 0 read_superblock omap replica is missing. 2024-09-16T09:46:21.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 235559 9.38 MB/sec execute 521 sec latency 55.506 ms 2024-09-16T09:46:21.829 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:21 smithi078 ceph-mon[63909]: pgmap v235: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.9 MiB/s wr, 486 op/s; 171/1596 objects degraded (10.714%) 2024-09-16T09:46:22.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:21 smithi088 ceph-mon[52758]: pgmap v235: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.9 MiB/s wr, 486 op/s; 171/1596 objects degraded (10.714%) 2024-09-16T09:46:22.051 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:21 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:46:21.850+0000 7fcc56755640 -1 osd.5 133 log_to_monitors true 2024-09-16T09:46:22.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 235812 9.37 MB/sec execute 522 sec latency 66.629 ms 2024-09-16T09:46:22.847 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:22 smithi078 ceph-mon[63909]: from='osd.5 [v2:172.21.15.88:6808/2532988575,v1:172.21.15.88:6809/2532988575]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-16T09:46:22.848 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:22 smithi078 ceph-mon[63909]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-16T09:46:23.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:22 smithi088 ceph-mon[52758]: from='osd.5 [v2:172.21.15.88:6808/2532988575,v1:172.21.15.88:6809/2532988575]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-16T09:46:23.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:22 smithi088 ceph-mon[52758]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-16T09:46:23.550 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:46:23 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:46:23.248+0000 7fcc4a4f7640 -1 osd.5 133 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T09:46:23.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 235978 9.36 MB/sec execute 523 sec latency 120.698 ms 2024-09-16T09:46:23.939 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:23 smithi078 ceph-mon[63909]: pgmap v236: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.2 MiB/s wr, 419 op/s; 176/1632 objects degraded (10.784%) 2024-09-16T09:46:23.940 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:23 smithi078 ceph-mon[63909]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-16T09:46:23.940 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:23 smithi078 ceph-mon[63909]: osdmap e136: 8 total, 7 up, 8 in 2024-09-16T09:46:23.940 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:23 smithi078 ceph-mon[63909]: from='osd.5 [v2:172.21.15.88:6808/2532988575,v1:172.21.15.88:6809/2532988575]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi088", "root=default"]} : dispatch 2024-09-16T09:46:23.940 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:23 smithi078 ceph-mon[63909]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi088", "root=default"]} : dispatch 2024-09-16T09:46:23.940 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:23 smithi078 ceph-mon[63909]: from='osd.5 ' entity='osd.5' 2024-09-16T09:46:24.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:23 smithi088 ceph-mon[52758]: pgmap v236: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.2 MiB/s wr, 419 op/s; 176/1632 objects degraded (10.784%) 2024-09-16T09:46:24.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:23 smithi088 ceph-mon[52758]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-16T09:46:24.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:23 smithi088 ceph-mon[52758]: osdmap e136: 8 total, 7 up, 8 in 2024-09-16T09:46:24.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:23 smithi088 ceph-mon[52758]: from='osd.5 [v2:172.21.15.88:6808/2532988575,v1:172.21.15.88:6809/2532988575]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi088", "root=default"]} : dispatch 2024-09-16T09:46:24.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:23 smithi088 ceph-mon[52758]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi088", "root=default"]} : dispatch 2024-09-16T09:46:24.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:23 smithi088 ceph-mon[52758]: from='osd.5 ' entity='osd.5' 2024-09-16T09:46:24.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 236149 9.35 MB/sec execute 524 sec latency 429.343 ms 2024-09-16T09:46:25.001 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:24 smithi078 ceph-mon[63909]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T09:46:25.001 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:24 smithi078 ceph-mon[63909]: osd.5 [v2:172.21.15.88:6808/2532988575,v1:172.21.15.88:6809/2532988575] boot 2024-09-16T09:46:25.001 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:24 smithi078 ceph-mon[63909]: osdmap e137: 8 total, 8 up, 8 in 2024-09-16T09:46:25.002 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:24 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-16T09:46:25.002 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:24 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 176/1632 objects degraded (10.784%), 23 pgs degraded (PG_DEGRADED) 2024-09-16T09:46:25.002 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:24 smithi078 ceph-mon[63909]: osdmap e138: 8 total, 8 up, 8 in 2024-09-16T09:46:25.201 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:24 smithi088 ceph-mon[52758]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T09:46:25.202 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:24 smithi088 ceph-mon[52758]: osd.5 [v2:172.21.15.88:6808/2532988575,v1:172.21.15.88:6809/2532988575] boot 2024-09-16T09:46:25.202 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:24 smithi088 ceph-mon[52758]: osdmap e137: 8 total, 8 up, 8 in 2024-09-16T09:46:25.202 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:24 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-16T09:46:25.202 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:24 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 176/1632 objects degraded (10.784%), 23 pgs degraded (PG_DEGRADED) 2024-09-16T09:46:25.202 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:24 smithi088 ceph-mon[52758]: osdmap e138: 8 total, 8 up, 8 in 2024-09-16T09:46:25.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 236354 9.34 MB/sec execute 525 sec latency 844.820 ms 2024-09-16T09:46:26.038 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:25 smithi078 ceph-mon[63909]: pgmap v239: 97 pgs: 3 peering, 8 active+undersized, 21 active+undersized+degraded, 65 active+clean; 352 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.7 MiB/s wr, 435 op/s; 155/1671 objects degraded (9.276%) 2024-09-16T09:46:26.038 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:25 smithi078 ceph-mon[63909]: osdmap e139: 8 total, 8 up, 8 in 2024-09-16T09:46:26.214 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:25 smithi088 ceph-mon[52758]: pgmap v239: 97 pgs: 3 peering, 8 active+undersized, 21 active+undersized+degraded, 65 active+clean; 352 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.7 MiB/s wr, 435 op/s; 155/1671 objects degraded (9.276%) 2024-09-16T09:46:26.214 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:25 smithi088 ceph-mon[52758]: osdmap e139: 8 total, 8 up, 8 in 2024-09-16T09:46:26.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 236641 9.34 MB/sec execute 526 sec latency 119.963 ms 2024-09-16T09:46:27.369 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:27 smithi078 ceph-mon[63909]: pgmap v242: 97 pgs: 3 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized, 19 active+undersized+degraded, 69 active+clean; 352 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 4.8 MiB/s wr, 509 op/s; 147/1626 objects degraded (9.041%); 41 KiB/s, 2 objects/s recovering 2024-09-16T09:46:27.369 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:27 smithi078 ceph-mon[63909]: osdmap e140: 8 total, 8 up, 8 in 2024-09-16T09:46:27.550 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:27 smithi088 ceph-mon[52758]: pgmap v242: 97 pgs: 3 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized, 19 active+undersized+degraded, 69 active+clean; 352 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 4.8 MiB/s wr, 509 op/s; 147/1626 objects degraded (9.041%); 41 KiB/s, 2 objects/s recovering 2024-09-16T09:46:27.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:27 smithi088 ceph-mon[52758]: osdmap e140: 8 total, 8 up, 8 in 2024-09-16T09:46:27.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 236901 9.34 MB/sec execute 527 sec latency 396.713 ms 2024-09-16T09:46:28.406 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:28 smithi078 ceph-mon[63909]: osdmap e141: 8 total, 8 up, 8 in 2024-09-16T09:46:28.406 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:28 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:28.407 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:28 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:46:28.407 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:28 smithi078 ceph-mon[63909]: osdmap e142: 8 total, 8 up, 8 in 2024-09-16T09:46:28.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:28 smithi088 ceph-mon[52758]: osdmap e141: 8 total, 8 up, 8 in 2024-09-16T09:46:28.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:28 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:28.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:28 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:46:28.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:28 smithi088 ceph-mon[52758]: osdmap e142: 8 total, 8 up, 8 in 2024-09-16T09:46:28.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 237216 9.33 MB/sec execute 528 sec latency 1034.965 ms 2024-09-16T09:46:29.443 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:29 smithi078 ceph-mon[63909]: pgmap v245: 97 pgs: 3 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 87 active+clean; 348 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 2.7 MiB/s wr, 438 op/s; 52/1629 objects degraded (3.192%); 395 KiB/s, 26 objects/s recovering 2024-09-16T09:46:29.443 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:29 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 52/1629 objects degraded (3.192%), 7 pgs degraded (PG_DEGRADED) 2024-09-16T09:46:29.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 237424 9.32 MB/sec execute 529 sec latency 49.999 ms 2024-09-16T09:46:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:29 smithi088 ceph-mon[52758]: pgmap v245: 97 pgs: 3 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 87 active+clean; 348 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 2.7 MiB/s wr, 438 op/s; 52/1629 objects degraded (3.192%); 395 KiB/s, 26 objects/s recovering 2024-09-16T09:46:29.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:29 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 52/1629 objects degraded (3.192%), 7 pgs degraded (PG_DEGRADED) 2024-09-16T09:46:30.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 237729 9.32 MB/sec execute 530 sec latency 65.745 ms 2024-09-16T09:46:31.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:31 smithi078 ceph-mon[63909]: pgmap v247: 97 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.6 MiB/s wr, 440 op/s; 8/1536 objects degraded (0.521%); 1.6 MiB/s, 39 objects/s recovering 2024-09-16T09:46:31.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 237911 9.31 MB/sec execute 531 sec latency 125.944 ms 2024-09-16T09:46:31.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:31 smithi088 ceph-mon[52758]: pgmap v247: 97 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.6 MiB/s wr, 440 op/s; 8/1536 objects degraded (0.521%); 1.6 MiB/s, 39 objects/s recovering 2024-09-16T09:46:32.573 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:32 smithi078 ceph-mon[63909]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8/1536 objects degraded (0.521%), 2 pgs degraded) 2024-09-16T09:46:32.573 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:32 smithi078 ceph-mon[63909]: Cluster is now healthy 2024-09-16T09:46:32.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 238218 9.31 MB/sec execute 532 sec latency 50.100 ms 2024-09-16T09:46:32.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:32 smithi088 ceph-mon[52758]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8/1536 objects degraded (0.521%), 2 pgs degraded) 2024-09-16T09:46:32.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:32 smithi088 ceph-mon[52758]: Cluster is now healthy 2024-09-16T09:46:33.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 238553 9.32 MB/sec execute 533 sec latency 43.541 ms 2024-09-16T09:46:33.592 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:33 smithi078 ceph-mon[63909]: pgmap v248: 97 pgs: 97 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 2.0 MiB/s wr, 221 op/s; 1.9 MiB/s, 31 objects/s recovering 2024-09-16T09:46:33.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:33 smithi088 ceph-mon[52758]: pgmap v248: 97 pgs: 97 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 2.0 MiB/s wr, 221 op/s; 1.9 MiB/s, 31 objects/s recovering 2024-09-16T09:46:34.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 238758 9.32 MB/sec execute 534 sec latency 86.368 ms 2024-09-16T09:46:34.614 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:34 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-16T09:46:34.675 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:34 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-16T09:46:35.385 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:35 smithi078 ceph-mon[63909]: pgmap v249: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 3.7 MiB/s wr, 317 op/s; 1.5 MiB/s, 24 objects/s recovering 2024-09-16T09:46:35.386 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:35 smithi078 ceph-mon[63909]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-16T09:46:35.386 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:35 smithi078 ceph-mon[63909]: Upgrade: osd.6 is safe to restart 2024-09-16T09:46:35.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 238996 9.31 MB/sec execute 535 sec latency 108.523 ms 2024-09-16T09:46:35.730 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:35 smithi088 ceph-mon[52758]: pgmap v249: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 3.7 MiB/s wr, 317 op/s; 1.5 MiB/s, 24 objects/s recovering 2024-09-16T09:46:35.731 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:35 smithi088 ceph-mon[52758]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-16T09:46:35.731 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:35 smithi088 ceph-mon[52758]: Upgrade: osd.6 is safe to restart 2024-09-16T09:46:36.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 239264 9.30 MB/sec execute 536 sec latency 83.121 ms 2024-09-16T09:46:36.616 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-16T09:46:36.934 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:36 smithi078 ceph-mon[63909]: Upgrade: Updating osd.6 2024-09-16T09:46:36.935 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:36 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:36.935 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:36 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-16T09:46:36.935 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:36 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:46:36.935 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:36 smithi078 ceph-mon[63909]: Deploying daemon osd.6 on smithi088 2024-09-16T09:46:37.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:36 smithi088 ceph-mon[52758]: Upgrade: Updating osd.6 2024-09-16T09:46:37.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:36 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:37.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:36 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-16T09:46:37.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:36 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:46:37.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:36 smithi088 ceph-mon[52758]: Deploying daemon osd.6 on smithi088 2024-09-16T09:46:37.564 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T09:46:37.565 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (8m) 97s ago 18m 16.2M - 0.27.0 11f11916f8cd 3ebb6832fbcc 2024-09-16T09:46:37.565 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (4m) 97s ago 18m 6899k - 19.3.0-4882-g66e33d6a 261f2dc7b20f 72965213139c 2024-09-16T09:46:37.565 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi088 smithi088 running (4m) 22s ago 16m 6891k - 19.3.0-4882-g66e33d6a 261f2dc7b20f eac0e6855b1b 2024-09-16T09:46:37.565 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (7m) 97s ago 17m 45.2M - dad864ee21e9 4a91b97e29b1 2024-09-16T09:46:37.565 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.hiotnn smithi078 running (11m) 97s ago 11m 38.1M - 17.2.0 e1d6a67b021e c43db7767ab9 2024-09-16T09:46:37.565 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi088.dbunob smithi088 running (11m) 22s ago 11m 18.0M - 17.2.0 e1d6a67b021e d235f1bfbb05 2024-09-16T09:46:37.565 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rrfxge smithi078 *:8443,9283,8765 running (7m) 97s ago 20m 519M - 19.3.0-4882-g66e33d6a 261f2dc7b20f 0d25f2f30f1e 2024-09-16T09:46:37.565 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi088.ncxfdz smithi088 *:8443,9283,8765 running (6m) 22s ago 16m 457M - 19.3.0-4882-g66e33d6a 261f2dc7b20f e2d03f701acd 2024-09-16T09:46:37.565 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (6m) 97s ago 20m 58.2M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f f5a3a0fec757 2024-09-16T09:46:37.565 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi088 smithi088 running (5m) 22s ago 16m 46.2M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f 2e552147327a 2024-09-16T09:46:37.565 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.wawosh smithi078 *:2049 running (11m) 97s ago 11m 280M - 4.0 e1d6a67b021e 2544ed9d67f5 2024-09-16T09:46:37.565 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi088.bnccmw smithi088 *:2049 running (11m) 22s ago 11m 82.1M - 4.0 e1d6a67b021e e606adb67049 2024-09-16T09:46:37.566 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (7m) 97s ago 17m 9.82M - 1.7.0 72c9c2088986 895ac3ccb2f3 2024-09-16T09:46:37.566 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi088 smithi088 *:9100 running (7m) 22s ago 16m 12.2M - 1.7.0 72c9c2088986 c84a2e50de53 2024-09-16T09:46:37.566 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (4m) 97s ago 16m 348M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 85008b499cb7 2024-09-16T09:46:37.566 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (3m) 97s ago 15m 271M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 12c137fcba70 2024-09-16T09:46:37.566 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (2m) 97s ago 15m 211M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 23a8b69c5c64 2024-09-16T09:46:37.566 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (106s) 97s ago 14m 11.2M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 00c9aa84d41d 2024-09-16T09:46:37.566 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi088 running (63s) 22s ago 14m 254M 2704M 19.3.0-4882-g66e33d6a 261f2dc7b20f 1d92e66b75d9 2024-09-16T09:46:37.566 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi088 running (27s) 22s ago 13m 11.4M 2704M 19.3.0-4882-g66e33d6a 261f2dc7b20f 43abc1197fe2 2024-09-16T09:46:37.566 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi088 running (13m) 22s ago 13m 480M 2704M 17.2.0 e1d6a67b021e 7c1459b1a2d9 2024-09-16T09:46:37.566 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi088 running (12m) 22s ago 12m 734M 2704M 17.2.0 e1d6a67b021e 37f619c2ddf9 2024-09-16T09:46:37.566 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (6m) 97s ago 16m 53.1M - 2.51.0 1d3b7f56885b 4c956ead9125 2024-09-16T09:46:37.580 INFO:teuthology.orchestra.run.smithi078.stdout: 5 239522 9.29 MB/sec execute 537 sec latency 67.707 ms 2024-09-16T09:46:37.957 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:37 smithi078 ceph-mon[63909]: pgmap v250: 97 pgs: 97 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 4.2 MiB/s wr, 414 op/s; 1.3 MiB/s, 21 objects/s recovering 2024-09-16T09:46:37.957 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:37 smithi078 ceph-mon[63909]: from='client.34382 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:46:37.957 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:37 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:38.025 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:37 smithi088 ceph-mon[52758]: pgmap v250: 97 pgs: 97 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 4.2 MiB/s wr, 414 op/s; 1.3 MiB/s, 21 objects/s recovering 2024-09-16T09:46:38.025 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:37 smithi088 ceph-mon[52758]: from='client.34382 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:46:38.025 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:37 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:38.089 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:46:38.089 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-16T09:46:38.089 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:46:38.090 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:46:38.090 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-16T09:46:38.090 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:46:38.090 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:46:38.090 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-16T09:46:38.090 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-16T09:46:38.090 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 6 2024-09-16T09:46:38.090 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:46:38.090 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-16T09:46:38.090 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:46:38.090 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:46:38.090 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-16T09:46:38.090 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-16T09:46:38.090 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 10 2024-09-16T09:46:38.091 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-16T09:46:38.091 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:46:38.581 INFO:teuthology.orchestra.run.smithi078.stdout: 5 239767 9.29 MB/sec execute 538 sec latency 57.607 ms 2024-09-16T09:46:38.586 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:46:38.586 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f", 2024-09-16T09:46:38.586 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-16T09:46:38.586 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T09:46:38.586 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-16T09:46:38.586 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-09-16T09:46:38.586 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-16T09:46:38.586 INFO:teuthology.orchestra.run.smithi078.stdout: "mon" 2024-09-16T09:46:38.586 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-16T09:46:38.586 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "12/23 daemons upgraded", 2024-09-16T09:46:38.586 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-09-16T09:46:38.586 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-16T09:46:38.587 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:46:38.974 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:38 smithi078 ceph-mon[63909]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:46:38.975 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:38 smithi078 ceph-mon[63909]: from='client.34390 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:46:38.975 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:38 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/3637575998' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:46:39.002 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:38 smithi088 ceph-mon[52758]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:46:39.002 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:38 smithi088 ceph-mon[52758]: from='client.34390 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:46:39.002 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:38 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/3637575998' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:46:39.202 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-16T09:46:39.581 INFO:teuthology.orchestra.run.smithi078.stdout: 5 240065 9.29 MB/sec execute 539 sec latency 66.290 ms 2024-09-16T09:46:39.665 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:39 smithi088 systemd[1]: Stopping Ceph osd.6 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:46:39.982 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:39 smithi078 ceph-mon[63909]: pgmap v251: 97 pgs: 97 active+clean; 350 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.9 MiB/s wr, 391 op/s; 1.0 MiB/s, 9 objects/s recovering 2024-09-16T09:46:39.983 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:39 smithi078 ceph-mon[63909]: from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:46:39.983 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:39 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/52083632' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:46:39.983 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:39 smithi078 ceph-mon[63909]: osd.6 marked itself down and dead 2024-09-16T09:46:40.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:39 smithi088 ceph-mon[52758]: pgmap v251: 97 pgs: 97 active+clean; 350 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.9 MiB/s wr, 391 op/s; 1.0 MiB/s, 9 objects/s recovering 2024-09-16T09:46:40.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:39 smithi088 ceph-mon[52758]: from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:46:40.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:39 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/52083632' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:46:40.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:39 smithi088 ceph-mon[52758]: osd.6 marked itself down and dead 2024-09-16T09:46:40.051 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:39 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[39368]: 2024-09-16T09:46:39.663+0000 7f9b88048700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T09:46:40.051 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:39 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[39368]: 2024-09-16T09:46:39.663+0000 7f9b88048700 -1 osd.6 142 *** Got signal Terminated *** 2024-09-16T09:46:40.051 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:39 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[39368]: 2024-09-16T09:46:39.663+0000 7f9b88048700 -1 osd.6 142 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T09:46:40.581 INFO:teuthology.orchestra.run.smithi078.stdout: 5 240202 9.28 MB/sec execute 540 sec latency 827.484 ms 2024-09-16T09:46:40.699 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:40 smithi088 podman[60210]: 2024-09-16 09:46:40.426810714 +0000 UTC m=+0.884450421 container died 7c1459b1a2d95e4302bd82ae20be331562c138fc6210e3f923d77b7aa0239d1b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , release=754, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, name=centos-stream, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, RELEASE=HEAD, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, ceph=True) 2024-09-16T09:46:41.032 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:40 smithi078 ceph-mon[63909]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T09:46:41.033 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:40 smithi078 ceph-mon[63909]: osdmap e143: 8 total, 7 up, 8 in 2024-09-16T09:46:41.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:40 smithi088 ceph-mon[52758]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T09:46:41.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:40 smithi088 ceph-mon[52758]: osdmap e143: 8 total, 7 up, 8 in 2024-09-16T09:46:41.051 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:40 smithi088 podman[60210]: 2024-09-16 09:46:40.9219837 +0000 UTC m=+1.379623400 container cleanup 7c1459b1a2d95e4302bd82ae20be331562c138fc6210e3f923d77b7aa0239d1b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, architecture=x86_64, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, distribution-scope=public, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, ceph=True) 2024-09-16T09:46:41.051 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:40 smithi088 bash[60210]: ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6 2024-09-16T09:46:41.551 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:41 smithi088 podman[60224]: 2024-09-16 09:46:41.17929152 +0000 UTC m=+0.756709072 container remove 7c1459b1a2d95e4302bd82ae20be331562c138fc6210e3f923d77b7aa0239d1b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, RELEASE=HEAD, release=754, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-16T09:46:41.581 INFO:teuthology.orchestra.run.smithi078.stdout: 5 240505 9.27 MB/sec execute 541 sec latency 1069.125 ms 2024-09-16T09:46:41.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:41 smithi078 ceph-mon[63909]: pgmap v253: 97 pgs: 10 stale+active+clean, 87 active+clean; 348 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.0 MiB/s wr, 390 op/s; 413 KiB/s, 0 objects/s recovering 2024-09-16T09:46:42.047 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:41 smithi078 ceph-mon[63909]: osdmap e144: 8 total, 7 up, 8 in 2024-09-16T09:46:42.048 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:41 smithi088 ceph-mon[52758]: pgmap v253: 97 pgs: 10 stale+active+clean, 87 active+clean; 348 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.0 MiB/s wr, 390 op/s; 413 KiB/s, 0 objects/s recovering 2024-09-16T09:46:42.048 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:41 smithi088 ceph-mon[52758]: osdmap e144: 8 total, 7 up, 8 in 2024-09-16T09:46:42.049 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:41 smithi088 podman[60313]: 2024-09-16 09:46:41.485273681 +0000 UTC m=+0.017020299 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:46:42.049 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:41 smithi088 podman[60313]: 2024-09-16 09:46:41.613560516 +0000 UTC m=+0.145307125 container create 067b9dde02149b42cd4e1063a9f943ae3e02b17f5c5074c08acc3f90410955b2 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-16T09:46:42.188 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:41 smithi088 podman[60313]: 2024-09-16 09:46:41.966689253 +0000 UTC m=+0.498435865 container init 067b9dde02149b42cd4e1063a9f943ae3e02b17f5c5074c08acc3f90410955b2 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True) 2024-09-16T09:46:42.188 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:41 smithi088 podman[60313]: 2024-09-16 09:46:41.970232192 +0000 UTC m=+0.501978803 container start 067b9dde02149b42cd4e1063a9f943ae3e02b17f5c5074c08acc3f90410955b2 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-16T09:46:42.188 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:42 smithi088 podman[60313]: 2024-09-16 09:46:42.041895808 +0000 UTC m=+0.573642417 container attach 067b9dde02149b42cd4e1063a9f943ae3e02b17f5c5074c08acc3f90410955b2 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-16T09:46:42.551 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:42 smithi088 podman[60313]: 2024-09-16 09:46:42.187302442 +0000 UTC m=+0.719049060 container died 067b9dde02149b42cd4e1063a9f943ae3e02b17f5c5074c08acc3f90410955b2 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-16T09:46:42.581 INFO:teuthology.orchestra.run.smithi078.stdout: 5 240887 9.27 MB/sec execute 542 sec latency 39.440 ms 2024-09-16T09:46:43.301 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:43 smithi088 podman[60336]: 2024-09-16 09:46:43.005147526 +0000 UTC m=+0.806808430 container remove 067b9dde02149b42cd4e1063a9f943ae3e02b17f5c5074c08acc3f90410955b2 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True) 2024-09-16T09:46:43.301 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:43 smithi088 systemd[1]: ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.6.service: Deactivated successfully. 2024-09-16T09:46:43.301 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:43 smithi088 systemd[1]: Stopped Ceph osd.6 for a70c765e-740d-11ef-bceb-c7b262605968. 2024-09-16T09:46:43.301 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:43 smithi088 systemd[1]: ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.6.service: Consumed 4min 3.091s CPU time. 2024-09-16T09:46:43.301 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:43 smithi088 systemd[1]: Starting Ceph osd.6 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:46:43.342 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:43 smithi078 ceph-mon[63909]: pgmap v255: 97 pgs: 14 peering, 5 stale+active+clean, 78 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.4 MiB/s wr, 413 op/s 2024-09-16T09:46:43.342 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:43 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:43.342 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:43 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:46:43.343 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:43 smithi078 ceph-mon[63909]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-09-16T09:46:43.581 INFO:teuthology.orchestra.run.smithi078.stdout: 5 241098 9.26 MB/sec execute 543 sec latency 52.856 ms 2024-09-16T09:46:43.609 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:43 smithi088 ceph-mon[52758]: pgmap v255: 97 pgs: 14 peering, 5 stale+active+clean, 78 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.4 MiB/s wr, 413 op/s 2024-09-16T09:46:43.609 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:43 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:43.609 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:43 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:46:43.609 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:43 smithi088 ceph-mon[52758]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-09-16T09:46:43.609 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:43 smithi088 podman[60440]: 2024-09-16 09:46:43.359229352 +0000 UTC m=+0.017969693 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:46:43.609 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:43 smithi088 podman[60440]: 2024-09-16 09:46:43.481668096 +0000 UTC m=+0.140408433 container create 9f51e6d1716699c91cae362c137da8e52927a09832f9c7c34960bd4d47bda074 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6-activate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-16T09:46:43.887 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:43 smithi088 podman[60440]: 2024-09-16 09:46:43.742656476 +0000 UTC m=+0.401396806 container init 9f51e6d1716699c91cae362c137da8e52927a09832f9c7c34960bd4d47bda074 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-16T09:46:43.887 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:43 smithi088 podman[60440]: 2024-09-16 09:46:43.746975372 +0000 UTC m=+0.405715715 container start 9f51e6d1716699c91cae362c137da8e52927a09832f9c7c34960bd4d47bda074 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6-activate, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-16T09:46:43.888 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:43 smithi088 podman[60440]: 2024-09-16 09:46:43.834624241 +0000 UTC m=+0.493364575 container attach 9f51e6d1716699c91cae362c137da8e52927a09832f9c7c34960bd4d47bda074 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_CLEAN=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-16T09:46:44.301 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:43 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6-activate[60455]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:46:44.301 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:43 smithi088 bash[60440]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:46:44.301 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:43 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6-activate[60455]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:46:44.301 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:43 smithi088 bash[60440]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:46:44.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:44 smithi078 ceph-mon[63909]: Health check failed: Degraded data redundancy: 114/1593 objects degraded (7.156%), 16 pgs degraded (PG_DEGRADED) 2024-09-16T09:46:44.581 INFO:teuthology.orchestra.run.smithi078.stdout: 5 241474 9.27 MB/sec execute 544 sec latency 53.046 ms 2024-09-16T09:46:44.676 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:44 smithi088 ceph-mon[52758]: Health check failed: Degraded data redundancy: 114/1593 objects degraded (7.156%), 16 pgs degraded (PG_DEGRADED) 2024-09-16T09:46:44.964 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:44 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6-activate[60455]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-16T09:46:44.964 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:44 smithi088 bash[60440]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-16T09:46:44.964 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:44 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6-activate[60455]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-16T09:46:44.964 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:44 smithi088 bash[60440]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-16T09:46:45.300 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:44 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6-activate[60455]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-16T09:46:45.300 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:44 smithi088 bash[60440]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-16T09:46:45.300 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:44 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6-activate[60455]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-16T09:46:45.300 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:44 smithi088 bash[60440]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-16T09:46:45.300 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:44 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6-activate[60455]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-16T09:46:45.301 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:44 smithi088 bash[60440]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-16T09:46:45.301 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:44 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6-activate[60455]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-16T09:46:45.301 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:44 smithi088 bash[60440]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-16T09:46:45.301 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:44 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6-activate[60455]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-16T09:46:45.301 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:44 smithi088 bash[60440]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-16T09:46:45.301 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:44 smithi088 podman[60440]: 2024-09-16 09:46:44.99628568 +0000 UTC m=+1.655026016 container died 9f51e6d1716699c91cae362c137da8e52927a09832f9c7c34960bd4d47bda074 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-16T09:46:45.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:45 smithi078 ceph-mon[63909]: pgmap v256: 97 pgs: 5 active+undersized, 14 peering, 16 active+undersized+degraded, 62 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 3.7 MiB/s wr, 358 op/s; 114/1593 objects degraded (7.156%) 2024-09-16T09:46:45.550 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:45 smithi088 ceph-mon[52758]: pgmap v256: 97 pgs: 5 active+undersized, 14 peering, 16 active+undersized+degraded, 62 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 3.7 MiB/s wr, 358 op/s; 114/1593 objects degraded (7.156%) 2024-09-16T09:46:45.581 INFO:teuthology.orchestra.run.smithi078.stdout: 5 241755 9.26 MB/sec execute 545 sec latency 43.765 ms 2024-09-16T09:46:45.866 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:45 smithi088 podman[60596]: 2024-09-16 09:46:45.612521226 +0000 UTC m=+0.609140135 container remove 9f51e6d1716699c91cae362c137da8e52927a09832f9c7c34960bd4d47bda074 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6-activate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-16T09:46:45.867 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:45 smithi088 podman[60654]: 2024-09-16 09:46:45.728849074 +0000 UTC m=+0.017322876 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:46:45.867 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:45 smithi088 podman[60654]: 2024-09-16 09:46:45.865889711 +0000 UTC m=+0.154363511 container create 31302f8f7d8fa187cf12027dd840736a144a86b3235f1a4acf89fc453b5e51c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-16T09:46:46.551 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:46 smithi088 podman[60654]: 2024-09-16 09:46:46.154606082 +0000 UTC m=+0.443079889 container init 31302f8f7d8fa187cf12027dd840736a144a86b3235f1a4acf89fc453b5e51c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-16T09:46:46.551 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:46 smithi088 podman[60654]: 2024-09-16 09:46:46.157938789 +0000 UTC m=+0.446412598 container start 31302f8f7d8fa187cf12027dd840736a144a86b3235f1a4acf89fc453b5e51c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-16T09:46:46.551 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:46 smithi088 ceph-osd[60674]: -- 172.21.15.88:0/4067503027 <== mon.1 v2:172.21.15.88:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5641c2c5c1e0 con 0x5641c2bf0000 2024-09-16T09:46:46.552 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:46 smithi088 bash[60654]: 31302f8f7d8fa187cf12027dd840736a144a86b3235f1a4acf89fc453b5e51c0 2024-09-16T09:46:46.552 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:46 smithi088 systemd[1]: Started Ceph osd.6 for a70c765e-740d-11ef-bceb-c7b262605968. 2024-09-16T09:46:46.581 INFO:teuthology.orchestra.run.smithi078.stdout: 5 242086 9.28 MB/sec execute 546 sec latency 67.715 ms 2024-09-16T09:46:47.451 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:47 smithi078 ceph-mon[63909]: pgmap v257: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.6 MiB/s wr, 268 op/s; 171/1599 objects degraded (10.694%) 2024-09-16T09:46:47.451 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:47 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:47.451 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:47 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:47.451 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:47 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:46:47.451 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:47 smithi078 ceph-mon[63909]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-09-16T09:46:47.581 INFO:teuthology.orchestra.run.smithi078.stdout: 5 242353 9.28 MB/sec execute 547 sec latency 51.132 ms 2024-09-16T09:46:47.613 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:47 smithi088 ceph-mon[52758]: pgmap v257: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.6 MiB/s wr, 268 op/s; 171/1599 objects degraded (10.694%) 2024-09-16T09:46:47.613 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:47 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:47.613 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:47 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:47.613 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:47 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:46:47.613 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:47 smithi088 ceph-mon[52758]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-09-16T09:46:48.051 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:47 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:46:47.789+0000 7fc1706d6640 -1 Falling back to public interface 2024-09-16T09:46:48.581 INFO:teuthology.orchestra.run.smithi078.stdout: 5 242646 9.27 MB/sec execute 548 sec latency 55.615 ms 2024-09-16T09:46:49.492 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:49 smithi078 ceph-mon[63909]: pgmap v258: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 3.3 MiB/s wr, 399 op/s; 171/1605 objects degraded (10.654%) 2024-09-16T09:46:49.550 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:49 smithi088 ceph-mon[52758]: pgmap v258: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 3.3 MiB/s wr, 399 op/s; 171/1605 objects degraded (10.654%) 2024-09-16T09:46:49.581 INFO:teuthology.orchestra.run.smithi078.stdout: 5 242907 9.26 MB/sec execute 549 sec latency 53.254 ms 2024-09-16T09:46:50.519 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:50 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 176/1680 objects degraded (10.476%), 25 pgs degraded (PG_DEGRADED) 2024-09-16T09:46:50.581 INFO:teuthology.orchestra.run.smithi078.stdout: 5 243086 9.25 MB/sec execute 550 sec latency 73.326 ms 2024-09-16T09:46:50.800 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:50 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 176/1680 objects degraded (10.476%), 25 pgs degraded (PG_DEGRADED) 2024-09-16T09:46:51.581 INFO:teuthology.orchestra.run.smithi078.stdout: 5 243325 9.24 MB/sec execute 551 sec latency 80.540 ms 2024-09-16T09:46:51.827 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:51 smithi078 ceph-mon[63909]: pgmap v259: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.0 MiB/s wr, 373 op/s; 176/1680 objects degraded (10.476%) 2024-09-16T09:46:51.827 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:51 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:51.827 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:51 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:51.828 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:51 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:51.828 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:51 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:51.919 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:51 smithi088 ceph-mon[52758]: pgmap v259: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.0 MiB/s wr, 373 op/s; 176/1680 objects degraded (10.476%) 2024-09-16T09:46:51.919 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:51 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:51.919 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:51 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:51.919 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:51 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:51.919 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:51 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:52.581 INFO:teuthology.orchestra.run.smithi078.stdout: 5 243605 9.24 MB/sec execute 552 sec latency 61.442 ms 2024-09-16T09:46:53.581 INFO:teuthology.orchestra.run.smithi078.stdout: 5 243909 9.24 MB/sec execute 553 sec latency 61.984 ms 2024-09-16T09:46:53.860 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:53 smithi078 ceph-mon[63909]: pgmap v260: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.1 MiB/s wr, 428 op/s; 173/1665 objects degraded (10.390%) 2024-09-16T09:46:53.887 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:53 smithi088 ceph-mon[52758]: pgmap v260: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.1 MiB/s wr, 428 op/s; 173/1665 objects degraded (10.390%) 2024-09-16T09:46:54.581 INFO:teuthology.orchestra.run.smithi078.stdout: 5 244273 9.23 MB/sec execute 554 sec latency 43.597 ms 2024-09-16T09:46:55.581 INFO:teuthology.orchestra.run.smithi078.stdout: 5 244593 9.23 MB/sec execute 555 sec latency 52.729 ms 2024-09-16T09:46:55.634 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:55 smithi078 ceph-mon[63909]: pgmap v261: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.6 MiB/s wr, 448 op/s; 177/1563 objects degraded (11.324%) 2024-09-16T09:46:55.634 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:55 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:55.635 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:55 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:55.635 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:55 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:46:55.635 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:55 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:46:55.635 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:55 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:55.635 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:55 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:55.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:55 smithi088 ceph-mon[52758]: pgmap v261: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.6 MiB/s wr, 448 op/s; 177/1563 objects degraded (11.324%) 2024-09-16T09:46:55.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:55 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:55.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:55 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:55.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:55 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:46:55.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:55 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:46:55.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:55 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:55.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:55 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:56.581 INFO:teuthology.orchestra.run.smithi078.stdout: 5 245038 9.24 MB/sec execute 556 sec latency 36.482 ms 2024-09-16T09:46:56.663 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:56 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:46:56.663 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:56 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:46:56.663 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:56 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:46:56.664 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:56 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:46:56.664 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:56 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-16T09:46:56.801 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:56 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:46:56.437+0000 7fc1706d6640 -1 osd.6 0 read_superblock omap replica is missing. 2024-09-16T09:46:56.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:56 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:46:56.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:56 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:46:56.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:56 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:46:56.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:56 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:46:56.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:56 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-16T09:46:57.584 INFO:teuthology.orchestra.run.smithi078.stdout: 5 245430 9.24 MB/sec execute 557 sec latency 41.574 ms 2024-09-16T09:46:57.675 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:57 smithi078 ceph-mon[63909]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-16T09:46:57.675 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:57 smithi078 ceph-mon[63909]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-16T09:46:57.675 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:57 smithi078 ceph-mon[63909]: pgmap v262: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.9 MiB/s wr, 404 op/s; 176/1557 objects degraded (11.304%) 2024-09-16T09:46:57.675 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:57 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 176/1557 objects degraded (11.304%), 25 pgs degraded (PG_DEGRADED) 2024-09-16T09:46:57.675 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:57 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:57.675 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:57 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:46:58.051 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:57 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:46:57.697+0000 7fc1706d6640 -1 osd.6 142 log_to_monitors true 2024-09-16T09:46:58.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:57 smithi088 ceph-mon[52758]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-16T09:46:58.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:57 smithi088 ceph-mon[52758]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-16T09:46:58.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:57 smithi088 ceph-mon[52758]: pgmap v262: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.9 MiB/s wr, 404 op/s; 176/1557 objects degraded (11.304%) 2024-09-16T09:46:58.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:57 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 176/1557 objects degraded (11.304%), 25 pgs degraded (PG_DEGRADED) 2024-09-16T09:46:58.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:57 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:46:58.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:57 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:46:58.581 INFO:teuthology.orchestra.run.smithi078.stdout: 5 245761 9.25 MB/sec execute 558 sec latency 54.269 ms 2024-09-16T09:46:58.691 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:58 smithi078 ceph-mon[63909]: from='osd.6 [v2:172.21.15.88:6816/2053790412,v1:172.21.15.88:6817/2053790412]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-16T09:46:58.691 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:58 smithi078 ceph-mon[63909]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-16T09:46:59.051 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:46:58 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:46:58.664+0000 7fc168480640 -1 osd.6 142 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T09:46:59.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:58 smithi088 ceph-mon[52758]: from='osd.6 [v2:172.21.15.88:6816/2053790412,v1:172.21.15.88:6817/2053790412]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-16T09:46:59.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:58 smithi088 ceph-mon[52758]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-16T09:46:59.581 INFO:teuthology.orchestra.run.smithi078.stdout: 5 245995 9.25 MB/sec execute 559 sec latency 56.853 ms 2024-09-16T09:46:59.717 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:59 smithi078 ceph-mon[63909]: pgmap v263: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 3.7 MiB/s wr, 400 op/s; 183/1596 objects degraded (11.466%) 2024-09-16T09:46:59.718 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:59 smithi078 ceph-mon[63909]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-16T09:46:59.718 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:59 smithi078 ceph-mon[63909]: osdmap e145: 8 total, 7 up, 8 in 2024-09-16T09:46:59.718 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:59 smithi078 ceph-mon[63909]: from='osd.6 [v2:172.21.15.88:6816/2053790412,v1:172.21.15.88:6817/2053790412]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi088", "root=default"]} : dispatch 2024-09-16T09:46:59.718 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:46:59 smithi078 ceph-mon[63909]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi088", "root=default"]} : dispatch 2024-09-16T09:47:00.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:59 smithi088 ceph-mon[52758]: pgmap v263: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 3.7 MiB/s wr, 400 op/s; 183/1596 objects degraded (11.466%) 2024-09-16T09:47:00.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:59 smithi088 ceph-mon[52758]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-16T09:47:00.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:59 smithi088 ceph-mon[52758]: osdmap e145: 8 total, 7 up, 8 in 2024-09-16T09:47:00.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:59 smithi088 ceph-mon[52758]: from='osd.6 [v2:172.21.15.88:6816/2053790412,v1:172.21.15.88:6817/2053790412]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi088", "root=default"]} : dispatch 2024-09-16T09:47:00.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:46:59 smithi088 ceph-mon[52758]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi088", "root=default"]} : dispatch 2024-09-16T09:47:00.581 INFO:teuthology.orchestra.run.smithi078.stdout: 5 246162 9.23 MB/sec execute 560 sec latency 922.760 ms 2024-09-16T09:47:00.910 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:00 smithi078 ceph-mon[63909]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T09:47:00.911 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:00 smithi078 ceph-mon[63909]: osd.6 [v2:172.21.15.88:6816/2053790412,v1:172.21.15.88:6817/2053790412] boot 2024-09-16T09:47:00.911 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:00 smithi078 ceph-mon[63909]: osdmap e146: 8 total, 8 up, 8 in 2024-09-16T09:47:00.911 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:00 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-16T09:47:00.911 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:00 smithi078 ceph-mon[63909]: osdmap e147: 8 total, 8 up, 8 in 2024-09-16T09:47:01.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:00 smithi088 ceph-mon[52758]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T09:47:01.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:00 smithi088 ceph-mon[52758]: osd.6 [v2:172.21.15.88:6816/2053790412,v1:172.21.15.88:6817/2053790412] boot 2024-09-16T09:47:01.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:00 smithi088 ceph-mon[52758]: osdmap e146: 8 total, 8 up, 8 in 2024-09-16T09:47:01.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:00 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-16T09:47:01.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:00 smithi088 ceph-mon[52758]: osdmap e147: 8 total, 8 up, 8 in 2024-09-16T09:47:01.581 INFO:teuthology.orchestra.run.smithi078.stdout: 5 246336 9.23 MB/sec execute 561 sec latency 1750.851 ms 2024-09-16T09:47:01.670 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:01 smithi078 ceph-mon[63909]: pgmap v266: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 346 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.9 MiB/s wr, 491 op/s; 188/1659 objects degraded (11.332%) 2024-09-16T09:47:01.671 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:01 smithi078 ceph-mon[63909]: osdmap e148: 8 total, 8 up, 8 in 2024-09-16T09:47:02.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:01 smithi088 ceph-mon[52758]: pgmap v266: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 346 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.9 MiB/s wr, 491 op/s; 188/1659 objects degraded (11.332%) 2024-09-16T09:47:02.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:01 smithi088 ceph-mon[52758]: osdmap e148: 8 total, 8 up, 8 in 2024-09-16T09:47:02.581 INFO:teuthology.orchestra.run.smithi078.stdout: 5 246532 9.21 MB/sec execute 562 sec latency 1861.346 ms 2024-09-16T09:47:02.915 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:02 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 157/1695 objects degraded (9.263%), 21 pgs degraded (PG_DEGRADED) 2024-09-16T09:47:02.915 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:02 smithi078 ceph-mon[63909]: osdmap e149: 8 total, 8 up, 8 in 2024-09-16T09:47:03.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:02 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 157/1695 objects degraded (9.263%), 21 pgs degraded (PG_DEGRADED) 2024-09-16T09:47:03.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:02 smithi088 ceph-mon[52758]: osdmap e149: 8 total, 8 up, 8 in 2024-09-16T09:47:03.581 INFO:teuthology.orchestra.run.smithi078.stdout: 5 246728 9.20 MB/sec execute 563 sec latency 895.228 ms 2024-09-16T09:47:03.980 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:03 smithi078 ceph-mon[63909]: pgmap v269: 97 pgs: 9 peering, 5 active+undersized, 21 active+undersized+degraded, 62 active+clean; 351 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s wr, 797 op/s; 157/1695 objects degraded (9.263%) 2024-09-16T09:47:03.981 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:03 smithi078 ceph-mon[63909]: osdmap e150: 8 total, 8 up, 8 in 2024-09-16T09:47:04.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:03 smithi088 ceph-mon[52758]: pgmap v269: 97 pgs: 9 peering, 5 active+undersized, 21 active+undersized+degraded, 62 active+clean; 351 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.5 MiB/s wr, 797 op/s; 157/1695 objects degraded (9.263%) 2024-09-16T09:47:04.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:03 smithi088 ceph-mon[52758]: osdmap e150: 8 total, 8 up, 8 in 2024-09-16T09:47:04.582 INFO:teuthology.orchestra.run.smithi078.stdout: 5 246945 9.20 MB/sec execute 564 sec latency 1022.042 ms 2024-09-16T09:47:05.039 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:04 smithi078 ceph-mon[63909]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 157/1695 objects degraded (9.263%), 21 pgs degraded) 2024-09-16T09:47:05.039 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:04 smithi078 ceph-mon[63909]: Cluster is now healthy 2024-09-16T09:47:05.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:04 smithi088 ceph-mon[52758]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 157/1695 objects degraded (9.263%), 21 pgs degraded) 2024-09-16T09:47:05.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:04 smithi088 ceph-mon[52758]: Cluster is now healthy 2024-09-16T09:47:05.585 INFO:teuthology.orchestra.run.smithi078.stdout: 5 247152 9.19 MB/sec execute 565 sec latency 90.932 ms 2024-09-16T09:47:05.857 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:05 smithi078 ceph-mon[63909]: pgmap v272: 97 pgs: 12 peering, 85 active+clean; 346 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 3.8 MiB/s wr, 330 op/s; 750 KiB/s, 37 objects/s recovering 2024-09-16T09:47:06.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:05 smithi088 ceph-mon[52758]: pgmap v272: 97 pgs: 12 peering, 85 active+clean; 346 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 3.8 MiB/s wr, 330 op/s; 750 KiB/s, 37 objects/s recovering 2024-09-16T09:47:06.581 INFO:teuthology.orchestra.run.smithi078.stdout: 5 247494 9.19 MB/sec execute 566 sec latency 58.645 ms 2024-09-16T09:47:07.581 INFO:teuthology.orchestra.run.smithi078.stdout: 5 247814 9.19 MB/sec execute 567 sec latency 71.940 ms 2024-09-16T09:47:07.914 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:07 smithi078 ceph-mon[63909]: pgmap v273: 97 pgs: 3 peering, 94 active+clean; 341 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 2.4 MiB/s wr, 244 op/s; 805 KiB/s, 46 keys/s, 39 objects/s recovering 2024-09-16T09:47:08.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:07 smithi088 ceph-mon[52758]: pgmap v273: 97 pgs: 3 peering, 94 active+clean; 341 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 2.4 MiB/s wr, 244 op/s; 805 KiB/s, 46 keys/s, 39 objects/s recovering 2024-09-16T09:47:08.582 INFO:teuthology.orchestra.run.smithi078.stdout: 5 248234 9.19 MB/sec execute 568 sec latency 38.006 ms 2024-09-16T09:47:09.582 INFO:teuthology.orchestra.run.smithi078.stdout: 5 248640 9.20 MB/sec execute 569 sec latency 38.325 ms 2024-09-16T09:47:09.699 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-16T09:47:09.942 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:09 smithi078 ceph-mon[63909]: pgmap v274: 97 pgs: 2 peering, 95 active+clean; 345 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 2.5 MiB/s wr, 287 op/s; 679 KiB/s, 39 keys/s, 33 objects/s recovering 2024-09-16T09:47:10.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:09 smithi088 ceph-mon[52758]: pgmap v274: 97 pgs: 2 peering, 95 active+clean; 345 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 2.5 MiB/s wr, 287 op/s; 679 KiB/s, 39 keys/s, 33 objects/s recovering 2024-09-16T09:47:10.586 INFO:teuthology.orchestra.run.smithi078.stdout: 5 249039 9.21 MB/sec execute 570 sec latency 37.153 ms 2024-09-16T09:47:10.693 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T09:47:10.693 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (8m) 2m ago 19m 16.2M - 0.27.0 11f11916f8cd 3ebb6832fbcc 2024-09-16T09:47:10.693 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (5m) 2m ago 19m 6899k - 19.3.0-4882-g66e33d6a 261f2dc7b20f 72965213139c 2024-09-16T09:47:10.693 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi088 smithi088 running (5m) 20s ago 17m 6891k - 19.3.0-4882-g66e33d6a 261f2dc7b20f eac0e6855b1b 2024-09-16T09:47:10.693 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (8m) 2m ago 17m 45.2M - dad864ee21e9 4a91b97e29b1 2024-09-16T09:47:10.693 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.hiotnn smithi078 running (12m) 2m ago 12m 38.1M - 17.2.0 e1d6a67b021e c43db7767ab9 2024-09-16T09:47:10.693 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi088.dbunob smithi088 running (12m) 20s ago 12m 18.1M - 17.2.0 e1d6a67b021e d235f1bfbb05 2024-09-16T09:47:10.693 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rrfxge smithi078 *:8443,9283,8765 running (7m) 2m ago 20m 519M - 19.3.0-4882-g66e33d6a 261f2dc7b20f 0d25f2f30f1e 2024-09-16T09:47:10.693 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi088.ncxfdz smithi088 *:8443,9283,8765 running (6m) 20s ago 17m 457M - 19.3.0-4882-g66e33d6a 261f2dc7b20f e2d03f701acd 2024-09-16T09:47:10.693 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (6m) 2m ago 20m 58.2M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f f5a3a0fec757 2024-09-16T09:47:10.693 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi088 smithi088 running (6m) 20s ago 17m 47.4M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f 2e552147327a 2024-09-16T09:47:10.694 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.wawosh smithi078 *:2049 running (11m) 2m ago 11m 280M - 4.0 e1d6a67b021e 2544ed9d67f5 2024-09-16T09:47:10.694 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi088.bnccmw smithi088 *:2049 running (11m) 20s ago 11m 84.2M - 4.0 e1d6a67b021e e606adb67049 2024-09-16T09:47:10.694 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (8m) 2m ago 17m 9.82M - 1.7.0 72c9c2088986 895ac3ccb2f3 2024-09-16T09:47:10.694 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi088 smithi088 *:9100 running (7m) 20s ago 17m 11.2M - 1.7.0 72c9c2088986 c84a2e50de53 2024-09-16T09:47:10.694 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (4m) 2m ago 16m 348M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 85008b499cb7 2024-09-16T09:47:10.694 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (3m) 2m ago 16m 271M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 12c137fcba70 2024-09-16T09:47:10.694 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (3m) 2m ago 15m 211M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 23a8b69c5c64 2024-09-16T09:47:10.694 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (2m) 2m ago 15m 11.2M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 00c9aa84d41d 2024-09-16T09:47:10.694 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi088 running (96s) 20s ago 14m 281M 2704M 19.3.0-4882-g66e33d6a 261f2dc7b20f 1d92e66b75d9 2024-09-16T09:47:10.694 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi088 running (60s) 20s ago 14m 245M 2704M 19.3.0-4882-g66e33d6a 261f2dc7b20f 43abc1197fe2 2024-09-16T09:47:10.694 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi088 running (24s) 20s ago 13m 14.5M 2704M 19.3.0-4882-g66e33d6a 261f2dc7b20f 31302f8f7d8f 2024-09-16T09:47:10.694 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi088 running (13m) 20s ago 13m 785M 2704M 17.2.0 e1d6a67b021e 37f619c2ddf9 2024-09-16T09:47:10.694 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (7m) 2m ago 17m 53.1M - 2.51.0 1d3b7f56885b 4c956ead9125 2024-09-16T09:47:10.956 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:10 smithi078 ceph-mon[63909]: from='client.34408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:47:11.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:10 smithi088 ceph-mon[52758]: from='client.34408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:47:11.358 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:47:11.358 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-16T09:47:11.358 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:47:11.358 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:47:11.358 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-16T09:47:11.358 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:47:11.358 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:47:11.358 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-16T09:47:11.358 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-16T09:47:11.358 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 7 2024-09-16T09:47:11.359 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:47:11.359 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-16T09:47:11.359 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:47:11.359 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:47:11.359 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-16T09:47:11.359 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-16T09:47:11.359 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 11 2024-09-16T09:47:11.359 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-16T09:47:11.359 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:47:11.586 INFO:teuthology.orchestra.run.smithi078.stdout: 5 249292 9.21 MB/sec execute 571 sec latency 74.628 ms 2024-09-16T09:47:11.896 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:47:11.896 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f", 2024-09-16T09:47:11.896 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-16T09:47:11.896 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T09:47:11.896 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-16T09:47:11.896 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-09-16T09:47:11.896 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-16T09:47:11.896 INFO:teuthology.orchestra.run.smithi078.stdout: "mon" 2024-09-16T09:47:11.896 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-16T09:47:11.896 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "13/23 daemons upgraded", 2024-09-16T09:47:11.897 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-09-16T09:47:11.897 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-16T09:47:11.897 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:47:11.988 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:11 smithi078 ceph-mon[63909]: pgmap v275: 97 pgs: 97 active+clean; 343 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 3.4 MiB/s wr, 351 op/s; 1.5 MiB/s, 31 keys/s, 26 objects/s recovering 2024-09-16T09:47:11.989 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:11 smithi078 ceph-mon[63909]: from='client.34412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:47:11.989 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:11 smithi078 ceph-mon[63909]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:47:11.989 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:11 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-16T09:47:11.989 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:11 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/947400616' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:47:12.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:11 smithi088 ceph-mon[52758]: pgmap v275: 97 pgs: 97 active+clean; 343 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 3.4 MiB/s wr, 351 op/s; 1.5 MiB/s, 31 keys/s, 26 objects/s recovering 2024-09-16T09:47:12.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:11 smithi088 ceph-mon[52758]: from='client.34412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:47:12.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:11 smithi088 ceph-mon[52758]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:47:12.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:11 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-16T09:47:12.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:11 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/947400616' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:47:12.562 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-16T09:47:12.586 INFO:teuthology.orchestra.run.smithi078.stdout: 5 249527 9.21 MB/sec execute 572 sec latency 57.580 ms 2024-09-16T09:47:12.999 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:12 smithi078 ceph-mon[63909]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-16T09:47:12.999 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:12 smithi078 ceph-mon[63909]: Upgrade: osd.7 is safe to restart 2024-09-16T09:47:13.000 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:12 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:13.000 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:12 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:47:13.000 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:12 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/593297593' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:47:13.000 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:12 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:13.000 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:12 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-16T09:47:13.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:12 smithi088 ceph-mon[52758]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-16T09:47:13.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:12 smithi088 ceph-mon[52758]: Upgrade: osd.7 is safe to restart 2024-09-16T09:47:13.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:12 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:13.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:12 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:47:13.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:12 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/593297593' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:47:13.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:12 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:13.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:12 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-16T09:47:13.586 INFO:teuthology.orchestra.run.smithi078.stdout: 5 249788 9.20 MB/sec execute 573 sec latency 49.770 ms 2024-09-16T09:47:14.040 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:13 smithi078 ceph-mon[63909]: from='client.34424 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:47:14.041 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:13 smithi078 ceph-mon[63909]: pgmap v276: 97 pgs: 97 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.4 MiB/s wr, 344 op/s; 1.3 MiB/s, 26 keys/s, 23 objects/s recovering 2024-09-16T09:47:14.041 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:13 smithi078 ceph-mon[63909]: Upgrade: Updating osd.7 2024-09-16T09:47:14.041 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:13 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:47:14.041 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:13 smithi078 ceph-mon[63909]: Deploying daemon osd.7 on smithi088 2024-09-16T09:47:14.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:13 smithi088 ceph-mon[52758]: from='client.34424 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:47:14.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:13 smithi088 ceph-mon[52758]: pgmap v276: 97 pgs: 97 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.4 MiB/s wr, 344 op/s; 1.3 MiB/s, 26 keys/s, 23 objects/s recovering 2024-09-16T09:47:14.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:13 smithi088 ceph-mon[52758]: Upgrade: Updating osd.7 2024-09-16T09:47:14.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:13 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:47:14.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:13 smithi088 ceph-mon[52758]: Deploying daemon osd.7 on smithi088 2024-09-16T09:47:14.586 INFO:teuthology.orchestra.run.smithi078.stdout: 5 250046 9.19 MB/sec execute 574 sec latency 65.465 ms 2024-09-16T09:47:15.586 INFO:teuthology.orchestra.run.smithi078.stdout: 5 250320 9.18 MB/sec execute 575 sec latency 69.789 ms 2024-09-16T09:47:15.834 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:15 smithi078 ceph-mon[63909]: pgmap v277: 97 pgs: 97 active+clean; 349 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.4 MiB/s wr, 407 op/s; 1009 KiB/s, 24 keys/s, 9 objects/s recovering 2024-09-16T09:47:16.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:15 smithi088 ceph-mon[52758]: pgmap v277: 97 pgs: 97 active+clean; 349 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.4 MiB/s wr, 407 op/s; 1009 KiB/s, 24 keys/s, 9 objects/s recovering 2024-09-16T09:47:16.051 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:15 smithi088 systemd[1]: Stopping Ceph osd.7 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:47:16.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:16 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[42344]: 2024-09-16T09:47:16.163+0000 7f3028f48700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T09:47:16.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:16 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[42344]: 2024-09-16T09:47:16.163+0000 7f3028f48700 -1 osd.7 150 *** Got signal Terminated *** 2024-09-16T09:47:16.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:16 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[42344]: 2024-09-16T09:47:16.163+0000 7f3028f48700 -1 osd.7 150 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T09:47:16.586 INFO:teuthology.orchestra.run.smithi078.stdout: 5 250561 9.18 MB/sec execute 576 sec latency 61.497 ms 2024-09-16T09:47:17.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:16 smithi088 ceph-mon[52758]: osd.7 marked itself down and dead 2024-09-16T09:47:17.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:16 smithi078 ceph-mon[63909]: osd.7 marked itself down and dead 2024-09-16T09:47:17.586 INFO:teuthology.orchestra.run.smithi078.stdout: 5 250599 9.16 MB/sec execute 577 sec latency 795.383 ms 2024-09-16T09:47:17.774 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:17 smithi088 podman[62230]: 2024-09-16 09:47:17.466814379 +0000 UTC m=+1.424232832 container died 37f619c2ddf91b8abf426475667f0d07a08fc145cb5807ca02a7fa6ef489ab3e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, version=8, vendor=Red Hat, Inc., vcs-type=git, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-16T09:47:18.049 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:17 smithi078 ceph-mon[63909]: pgmap v278: 97 pgs: 97 active+clean; 351 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.2 MiB/s wr, 426 op/s; 792 KiB/s, 20 keys/s, 5 objects/s recovering 2024-09-16T09:47:18.050 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:17 smithi078 ceph-mon[63909]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T09:47:18.050 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:17 smithi078 ceph-mon[63909]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-16T09:47:18.050 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:17 smithi078 ceph-mon[63909]: osdmap e151: 8 total, 7 up, 8 in 2024-09-16T09:47:18.050 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:17 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:18.050 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:17 smithi088 podman[62230]: 2024-09-16 09:47:17.872269905 +0000 UTC m=+1.829688367 container cleanup 37f619c2ddf91b8abf426475667f0d07a08fc145cb5807ca02a7fa6ef489ab3e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, version=8, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, release=754, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-16T09:47:18.051 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:17 smithi088 bash[62230]: ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7 2024-09-16T09:47:18.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:17 smithi088 ceph-mon[52758]: pgmap v278: 97 pgs: 97 active+clean; 351 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.2 MiB/s wr, 426 op/s; 792 KiB/s, 20 keys/s, 5 objects/s recovering 2024-09-16T09:47:18.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:17 smithi088 ceph-mon[52758]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-16T09:47:18.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:17 smithi088 ceph-mon[52758]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-16T09:47:18.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:17 smithi088 ceph-mon[52758]: osdmap e151: 8 total, 7 up, 8 in 2024-09-16T09:47:18.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:17 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:18.546 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:18 smithi088 podman[62255]: 2024-09-16 09:47:18.099049834 +0000 UTC m=+0.634904962 container remove 37f619c2ddf91b8abf426475667f0d07a08fc145cb5807ca02a7fa6ef489ab3e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7, ceph=True, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, name=centos-stream, vcs-type=git, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., RELEASE=HEAD, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, maintainer=Guillaume Abrioux , io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870) 2024-09-16T09:47:18.586 INFO:teuthology.orchestra.run.smithi078.stdout: 5 250900 9.16 MB/sec execute 578 sec latency 1081.735 ms 2024-09-16T09:47:18.801 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:18 smithi088 podman[62345]: 2024-09-16 09:47:18.448235219 +0000 UTC m=+0.016215659 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:47:18.801 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:18 smithi088 podman[62345]: 2024-09-16 09:47:18.549891687 +0000 UTC m=+0.117872110 container create 5fca97f01d36f7406fa2c076ab1271bd024c09d334af9b03f0fe45e14926edba (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-16T09:47:18.801 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:18 smithi088 podman[62345]: 2024-09-16 09:47:18.778616973 +0000 UTC m=+0.346597396 container init 5fca97f01d36f7406fa2c076ab1271bd024c09d334af9b03f0fe45e14926edba (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7-deactivate, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-16T09:47:18.802 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:18 smithi088 podman[62345]: 2024-09-16 09:47:18.782022073 +0000 UTC m=+0.350002497 container start 5fca97f01d36f7406fa2c076ab1271bd024c09d334af9b03f0fe45e14926edba (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS) 2024-09-16T09:47:18.833 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:18 smithi078 ceph-mon[63909]: osdmap e152: 8 total, 7 up, 8 in 2024-09-16T09:47:19.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:18 smithi088 ceph-mon[52758]: osdmap e152: 8 total, 7 up, 8 in 2024-09-16T09:47:19.301 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:18 smithi088 podman[62345]: 2024-09-16 09:47:18.879004328 +0000 UTC m=+0.446984764 container attach 5fca97f01d36f7406fa2c076ab1271bd024c09d334af9b03f0fe45e14926edba (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-16T09:47:19.301 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:18 smithi088 podman[62345]: 2024-09-16 09:47:18.989444501 +0000 UTC m=+0.557424925 container died 5fca97f01d36f7406fa2c076ab1271bd024c09d334af9b03f0fe45e14926edba (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-16T09:47:19.586 INFO:teuthology.orchestra.run.smithi078.stdout: 5 251316 9.16 MB/sec execute 579 sec latency 46.949 ms 2024-09-16T09:47:19.861 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:19 smithi078 ceph-mon[63909]: pgmap v281: 97 pgs: 2 active+undersized, 11 stale+active+clean, 6 active+undersized+degraded, 78 active+clean; 347 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.4 MiB/s wr, 453 op/s; 44/1578 objects degraded (2.788%) 2024-09-16T09:47:19.862 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:19 smithi078 ceph-mon[63909]: Health check failed: Degraded data redundancy: 44/1578 objects degraded (2.788%), 6 pgs degraded (PG_DEGRADED) 2024-09-16T09:47:20.055 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:19 smithi088 ceph-mon[52758]: pgmap v281: 97 pgs: 2 active+undersized, 11 stale+active+clean, 6 active+undersized+degraded, 78 active+clean; 347 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.4 MiB/s wr, 453 op/s; 44/1578 objects degraded (2.788%) 2024-09-16T09:47:20.056 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:19 smithi088 ceph-mon[52758]: Health check failed: Degraded data redundancy: 44/1578 objects degraded (2.788%), 6 pgs degraded (PG_DEGRADED) 2024-09-16T09:47:20.056 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:19 smithi088 podman[62368]: 2024-09-16 09:47:19.80053921 +0000 UTC m=+0.803227044 container remove 5fca97f01d36f7406fa2c076ab1271bd024c09d334af9b03f0fe45e14926edba (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-16T09:47:20.056 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:19 smithi088 systemd[1]: ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.7.service: Deactivated successfully. 2024-09-16T09:47:20.056 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:19 smithi088 systemd[1]: Stopped Ceph osd.7 for a70c765e-740d-11ef-bceb-c7b262605968. 2024-09-16T09:47:20.056 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:19 smithi088 systemd[1]: ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.7.service: Consumed 6min 24.876s CPU time. 2024-09-16T09:47:20.319 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:20 smithi088 systemd[1]: Starting Ceph osd.7 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:47:20.320 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:20 smithi088 podman[62462]: 2024-09-16 09:47:20.195752791 +0000 UTC m=+0.020248611 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:47:20.320 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:20 smithi088 podman[62462]: 2024-09-16 09:47:20.318620932 +0000 UTC m=+0.143116733 container create 07e832f3f16d198a39e0548515c14c11ee7f8295ec04cd4c12656d734d066ea0 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-16T09:47:20.586 INFO:teuthology.orchestra.run.smithi078.stdout: 5 251684 9.16 MB/sec execute 580 sec latency 52.923 ms 2024-09-16T09:47:21.051 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:20 smithi088 podman[62462]: 2024-09-16 09:47:20.571302443 +0000 UTC m=+0.395798244 container init 07e832f3f16d198a39e0548515c14c11ee7f8295ec04cd4c12656d734d066ea0 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD) 2024-09-16T09:47:21.051 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:20 smithi088 podman[62462]: 2024-09-16 09:47:20.575768409 +0000 UTC m=+0.400264208 container start 07e832f3f16d198a39e0548515c14c11ee7f8295ec04cd4c12656d734d066ea0 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6) 2024-09-16T09:47:21.052 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:20 smithi088 podman[62462]: 2024-09-16 09:47:20.663338854 +0000 UTC m=+0.487834679 container attach 07e832f3f16d198a39e0548515c14c11ee7f8295ec04cd4c12656d734d066ea0 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7-activate, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True) 2024-09-16T09:47:21.052 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:20 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7-activate[62477]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:47:21.052 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:20 smithi088 bash[62462]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:47:21.052 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:20 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7-activate[62477]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:47:21.052 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:20 smithi088 bash[62462]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-16T09:47:21.525 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:21 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7-activate[62477]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-16T09:47:21.525 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:21 smithi088 bash[62462]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-16T09:47:21.525 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:21 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7-activate[62477]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-16T09:47:21.525 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:21 smithi088 bash[62462]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-16T09:47:21.525 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:21 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7-activate[62477]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-16T09:47:21.526 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:21 smithi088 bash[62462]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-16T09:47:21.586 INFO:teuthology.orchestra.run.smithi078.stdout: 5 252039 9.17 MB/sec execute 581 sec latency 78.997 ms 2024-09-16T09:47:21.801 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:21 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7-activate[62477]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-16T09:47:21.801 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:21 smithi088 bash[62462]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-16T09:47:21.801 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:21 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7-activate[62477]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-16T09:47:21.801 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:21 smithi088 bash[62462]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-16T09:47:21.801 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:21 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7-activate[62477]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-16T09:47:21.801 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:21 smithi088 bash[62462]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-16T09:47:21.802 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:21 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7-activate[62477]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-16T09:47:21.802 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:21 smithi088 bash[62462]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-16T09:47:21.802 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:21 smithi088 podman[62462]: 2024-09-16 09:47:21.560095959 +0000 UTC m=+1.384591751 container died 07e832f3f16d198a39e0548515c14c11ee7f8295ec04cd4c12656d734d066ea0 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS) 2024-09-16T09:47:21.904 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:21 smithi078 ceph-mon[63909]: pgmap v282: 97 pgs: 7 active+undersized, 6 stale+active+clean, 23 active+undersized+degraded, 61 active+clean; 343 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.3 MiB/s wr, 439 op/s; 138/1497 objects degraded (9.218%) 2024-09-16T09:47:22.161 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:21 smithi088 ceph-mon[52758]: pgmap v282: 97 pgs: 7 active+undersized, 6 stale+active+clean, 23 active+undersized+degraded, 61 active+clean; 343 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.3 MiB/s wr, 439 op/s; 138/1497 objects degraded (9.218%) 2024-09-16T09:47:22.420 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:22 smithi088 podman[62617]: 2024-09-16 09:47:22.16009192 +0000 UTC m=+0.593375393 container remove 07e832f3f16d198a39e0548515c14c11ee7f8295ec04cd4c12656d734d066ea0 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7-activate, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True) 2024-09-16T09:47:22.420 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:22 smithi088 podman[62673]: 2024-09-16 09:47:22.297822291 +0000 UTC m=+0.019040658 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:47:22.420 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:22 smithi088 podman[62673]: 2024-09-16 09:47:22.419498204 +0000 UTC m=+0.140716570 container create f3768fef8040c16c038f02f715c680dd3c64556d3095dd4b785c6ef996fe3091 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS) 2024-09-16T09:47:22.586 INFO:teuthology.orchestra.run.smithi078.stdout: 5 252436 9.18 MB/sec execute 582 sec latency 47.431 ms 2024-09-16T09:47:23.301 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:22 smithi088 podman[62673]: 2024-09-16 09:47:22.834188793 +0000 UTC m=+0.555407158 container init f3768fef8040c16c038f02f715c680dd3c64556d3095dd4b785c6ef996fe3091 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-16T09:47:23.301 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:22 smithi088 podman[62673]: 2024-09-16 09:47:22.837847356 +0000 UTC m=+0.559065722 container start f3768fef8040c16c038f02f715c680dd3c64556d3095dd4b785c6ef996fe3091 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-16T09:47:23.301 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:22 smithi088 ceph-osd[62692]: -- 172.21.15.88:0/2916670020 <== mon.1 v2:172.21.15.88:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55648b7561e0 con 0x55648b6e2000 2024-09-16T09:47:23.301 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:22 smithi088 bash[62673]: f3768fef8040c16c038f02f715c680dd3c64556d3095dd4b785c6ef996fe3091 2024-09-16T09:47:23.301 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:22 smithi088 systemd[1]: Started Ceph osd.7 for a70c765e-740d-11ef-bceb-c7b262605968. 2024-09-16T09:47:23.586 INFO:teuthology.orchestra.run.smithi078.stdout: 5 252812 9.18 MB/sec execute 583 sec latency 44.389 ms 2024-09-16T09:47:23.940 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:23 smithi078 ceph-mon[63909]: pgmap v283: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 344 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s wr, 372 op/s; 238/1521 objects degraded (15.648%); 258 B/s, 0 objects/s recovering 2024-09-16T09:47:23.941 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:23 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:23.941 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:23 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:23.941 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:23 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:47:23.952 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:23 smithi088 ceph-mon[52758]: pgmap v283: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 344 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s wr, 372 op/s; 238/1521 objects degraded (15.648%); 258 B/s, 0 objects/s recovering 2024-09-16T09:47:23.952 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:23 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:23.952 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:23 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:23.952 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:23 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:47:24.587 INFO:teuthology.orchestra.run.smithi078.stdout: 5 253137 9.18 MB/sec execute 584 sec latency 30.020 ms 2024-09-16T09:47:24.968 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:24 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 238/1521 objects degraded (15.648%), 35 pgs degraded (PG_DEGRADED) 2024-09-16T09:47:25.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:24 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 238/1521 objects degraded (15.648%), 35 pgs degraded (PG_DEGRADED) 2024-09-16T09:47:25.051 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:24 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:47:24.725+0000 7ff79177c640 -1 Falling back to public interface 2024-09-16T09:47:25.587 INFO:teuthology.orchestra.run.smithi078.stdout: 5 253622 9.18 MB/sec execute 585 sec latency 30.805 ms 2024-09-16T09:47:25.973 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:25 smithi078 ceph-mon[63909]: pgmap v284: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 349 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.8 MiB/s wr, 535 op/s; 263/1713 objects degraded (15.353%); 258 B/s, 0 objects/s recovering 2024-09-16T09:47:26.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:25 smithi088 ceph-mon[52758]: pgmap v284: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 349 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.8 MiB/s wr, 535 op/s; 263/1713 objects degraded (15.353%); 258 B/s, 0 objects/s recovering 2024-09-16T09:47:26.587 INFO:teuthology.orchestra.run.smithi078.stdout: 5 254157 9.18 MB/sec execute 586 sec latency 45.740 ms 2024-09-16T09:47:27.587 INFO:teuthology.orchestra.run.smithi078.stdout: 5 254474 9.18 MB/sec execute 587 sec latency 73.947 ms 2024-09-16T09:47:28.014 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:27 smithi078 ceph-mon[63909]: pgmap v285: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 353 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.8 MiB/s wr, 443 op/s; 274/1722 objects degraded (15.912%); 223 B/s, 0 objects/s recovering 2024-09-16T09:47:28.014 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:27 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:28.014 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:27 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:28.014 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:27 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:28.015 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:27 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:47:28.015 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:27 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:28.015 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:27 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:28.176 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:27 smithi088 ceph-mon[52758]: pgmap v285: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 353 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.8 MiB/s wr, 443 op/s; 274/1722 objects degraded (15.912%); 223 B/s, 0 objects/s recovering 2024-09-16T09:47:28.176 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:27 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:28.176 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:27 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:28.176 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:27 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:28.177 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:27 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:47:28.177 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:27 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:28.177 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:27 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:28.587 INFO:teuthology.orchestra.run.smithi078.stdout: 5 254861 9.18 MB/sec execute 588 sec latency 36.630 ms 2024-09-16T09:47:29.587 INFO:teuthology.orchestra.run.smithi078.stdout: 5 255164 9.18 MB/sec execute 589 sec latency 64.694 ms 2024-09-16T09:47:30.004 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:29 smithi088 ceph-mon[52758]: pgmap v286: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 346 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.7 MiB/s wr, 441 op/s; 265/1665 objects degraded (15.916%); 202 B/s, 0 objects/s recovering 2024-09-16T09:47:30.004 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:29 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 265/1665 objects degraded (15.916%), 35 pgs degraded (PG_DEGRADED) 2024-09-16T09:47:30.112 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:29 smithi078 ceph-mon[63909]: pgmap v286: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 346 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.7 MiB/s wr, 441 op/s; 265/1665 objects degraded (15.916%); 202 B/s, 0 objects/s recovering 2024-09-16T09:47:30.112 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:29 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 265/1665 objects degraded (15.916%), 35 pgs degraded (PG_DEGRADED) 2024-09-16T09:47:30.587 INFO:teuthology.orchestra.run.smithi078.stdout: 5 255528 9.19 MB/sec execute 590 sec latency 37.352 ms 2024-09-16T09:47:31.587 INFO:teuthology.orchestra.run.smithi078.stdout: 5 255865 9.19 MB/sec execute 591 sec latency 57.062 ms 2024-09-16T09:47:31.900 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:31 smithi078 ceph-mon[63909]: pgmap v287: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 347 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.8 MiB/s wr, 445 op/s; 259/1614 objects degraded (16.047%); 172 B/s, 0 objects/s recovering 2024-09-16T09:47:32.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:31 smithi088 ceph-mon[52758]: pgmap v287: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 347 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.8 MiB/s wr, 445 op/s; 259/1614 objects degraded (16.047%); 172 B/s, 0 objects/s recovering 2024-09-16T09:47:32.587 INFO:teuthology.orchestra.run.smithi078.stdout: 5 256152 9.20 MB/sec execute 592 sec latency 58.065 ms 2024-09-16T09:47:33.176 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:33.176 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:33.176 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:47:33.176 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:47:33.177 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:33.177 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: pgmap v288: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.3 MiB/s wr, 518 op/s; 254/1578 objects degraded (16.096%); 172 B/s, 0 objects/s recovering 2024-09-16T09:47:33.177 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:33.177 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:47:33.177 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:47:33.177 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:47:33.177 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:47:33.177 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:47:33.177 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: Upgrade: Setting container_image for all osd 2024-09-16T09:47:33.177 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:33.177 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-16T09:47:33.177 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-16T09:47:33.177 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-16T09:47:33.178 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-16T09:47:33.178 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-16T09:47:33.178 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-16T09:47:33.178 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-16T09:47:33.178 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-16T09:47:33.178 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-16T09:47:33.178 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-16T09:47:33.178 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-16T09:47:33.178 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-16T09:47:33.178 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-16T09:47:33.178 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-16T09:47:33.178 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-16T09:47:33.178 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-16T09:47:33.178 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: Upgrade: Setting require_osd_release to 19 squid 2024-09-16T09:47:33.179 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-16T09:47:33.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:33.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:33.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:47:33.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:47:33.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:33.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: pgmap v288: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.3 MiB/s wr, 518 op/s; 254/1578 objects degraded (16.096%); 172 B/s, 0 objects/s recovering 2024-09-16T09:47:33.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:33.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:47:33.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:47:33.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:47:33.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:47:33.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:47:33.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: Upgrade: Setting container_image for all osd 2024-09-16T09:47:33.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:33.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-16T09:47:33.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-16T09:47:33.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-16T09:47:33.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-16T09:47:33.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-16T09:47:33.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-16T09:47:33.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-16T09:47:33.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-16T09:47:33.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-16T09:47:33.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-16T09:47:33.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-16T09:47:33.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-16T09:47:33.303 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-16T09:47:33.304 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-16T09:47:33.304 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-16T09:47:33.304 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-16T09:47:33.304 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: Upgrade: Setting require_osd_release to 19 squid 2024-09-16T09:47:33.304 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-16T09:47:33.587 INFO:teuthology.orchestra.run.smithi078.stdout: 5 256421 9.19 MB/sec execute 593 sec latency 47.989 ms 2024-09-16T09:47:34.202 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:34 smithi078 ceph-mon[63909]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-16T09:47:34.202 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:34 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-16T09:47:34.202 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:34 smithi078 ceph-mon[63909]: osdmap e153: 8 total, 7 up, 8 in 2024-09-16T09:47:34.202 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:34 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi078.hiotnn"]} : dispatch 2024-09-16T09:47:34.202 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:34 smithi078 ceph-mon[63909]: Upgrade: It appears safe to stop mds.foofs.smithi078.hiotnn 2024-09-16T09:47:34.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:34 smithi088 ceph-mon[52758]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-16T09:47:34.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:34 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-16T09:47:34.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:34 smithi088 ceph-mon[52758]: osdmap e153: 8 total, 7 up, 8 in 2024-09-16T09:47:34.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:34 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi078.hiotnn"]} : dispatch 2024-09-16T09:47:34.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:34 smithi088 ceph-mon[52758]: Upgrade: It appears safe to stop mds.foofs.smithi078.hiotnn 2024-09-16T09:47:34.587 INFO:teuthology.orchestra.run.smithi078.stdout: 5 256618 9.19 MB/sec execute 594 sec latency 61.461 ms 2024-09-16T09:47:35.043 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:34 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:47:34.604+0000 7ff79177c640 -1 osd.7 0 read_superblock omap replica is missing. 2024-09-16T09:47:35.217 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:35 smithi078 ceph-mon[63909]: pgmap v290: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 350 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.5 MiB/s wr, 515 op/s; 264/1671 objects degraded (15.799%) 2024-09-16T09:47:35.217 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:35 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 254/1578 objects degraded (16.096%), 35 pgs degraded (PG_DEGRADED) 2024-09-16T09:47:35.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:35 smithi088 ceph-mon[52758]: pgmap v290: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 350 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.5 MiB/s wr, 515 op/s; 264/1671 objects degraded (15.799%) 2024-09-16T09:47:35.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:35 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 254/1578 objects degraded (16.096%), 35 pgs degraded (PG_DEGRADED) 2024-09-16T09:47:35.587 INFO:teuthology.orchestra.run.smithi078.stdout: 5 256885 9.18 MB/sec execute 595 sec latency 61.242 ms 2024-09-16T09:47:36.550 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:36 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:47:36.112+0000 7ff79177c640 -1 osd.7 150 log_to_monitors true 2024-09-16T09:47:36.587 INFO:teuthology.orchestra.run.smithi078.stdout: 5 257127 9.17 MB/sec execute 596 sec latency 70.117 ms 2024-09-16T09:47:37.014 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:36 smithi078 ceph-mon[63909]: Upgrade: Updating mds.foofs.smithi078.hiotnn 2024-09-16T09:47:37.014 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:36 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:37.014 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:36 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.hiotnn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-16T09:47:37.014 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:36 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:47:37.014 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:36 smithi078 ceph-mon[63909]: Deploying daemon mds.foofs.smithi078.hiotnn on smithi078 2024-09-16T09:47:37.014 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:36 smithi078 ceph-mon[63909]: from='osd.7 [v2:172.21.15.88:6824/2429727730,v1:172.21.15.88:6825/2429727730]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-16T09:47:37.014 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:36 smithi078 ceph-mon[63909]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-16T09:47:37.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:36 smithi088 ceph-mon[52758]: Upgrade: Updating mds.foofs.smithi078.hiotnn 2024-09-16T09:47:37.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:36 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:37.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:36 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.hiotnn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-16T09:47:37.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:36 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:47:37.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:36 smithi088 ceph-mon[52758]: Deploying daemon mds.foofs.smithi078.hiotnn on smithi078 2024-09-16T09:47:37.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:36 smithi088 ceph-mon[52758]: from='osd.7 [v2:172.21.15.88:6824/2429727730,v1:172.21.15.88:6825/2429727730]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-16T09:47:37.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:36 smithi088 ceph-mon[52758]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-16T09:47:37.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:47:37 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:47:37.265+0000 7ff78551e640 -1 osd.7 150 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T09:47:37.587 INFO:teuthology.orchestra.run.smithi078.stdout: 5 257376 9.16 MB/sec execute 597 sec latency 109.565 ms 2024-09-16T09:47:38.038 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:37 smithi078 ceph-mon[63909]: pgmap v291: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 353 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.1 MiB/s wr, 480 op/s; 263/1695 objects degraded (15.516%) 2024-09-16T09:47:38.038 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:37 smithi078 ceph-mon[63909]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-16T09:47:38.039 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:37 smithi078 ceph-mon[63909]: from='osd.7 [v2:172.21.15.88:6824/2429727730,v1:172.21.15.88:6825/2429727730]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi088", "root=default"]} : dispatch 2024-09-16T09:47:38.039 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:37 smithi078 ceph-mon[63909]: osdmap e154: 8 total, 7 up, 8 in 2024-09-16T09:47:38.039 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:37 smithi078 ceph-mon[63909]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi088", "root=default"]} : dispatch 2024-09-16T09:47:38.039 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:37 smithi078 ceph-mon[63909]: from='osd.7 ' entity='osd.7' 2024-09-16T09:47:38.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:37 smithi088 ceph-mon[52758]: pgmap v291: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 353 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.1 MiB/s wr, 480 op/s; 263/1695 objects degraded (15.516%) 2024-09-16T09:47:38.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:37 smithi088 ceph-mon[52758]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-16T09:47:38.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:37 smithi088 ceph-mon[52758]: from='osd.7 [v2:172.21.15.88:6824/2429727730,v1:172.21.15.88:6825/2429727730]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi088", "root=default"]} : dispatch 2024-09-16T09:47:38.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:37 smithi088 ceph-mon[52758]: osdmap e154: 8 total, 7 up, 8 in 2024-09-16T09:47:38.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:37 smithi088 ceph-mon[52758]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi088", "root=default"]} : dispatch 2024-09-16T09:47:38.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:37 smithi088 ceph-mon[52758]: from='osd.7 ' entity='osd.7' 2024-09-16T09:47:38.587 INFO:teuthology.orchestra.run.smithi078.stdout: 5 257640 9.16 MB/sec execute 598 sec latency 308.886 ms 2024-09-16T09:47:39.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:38 smithi088 ceph-mon[52758]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T09:47:39.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:38 smithi088 ceph-mon[52758]: osd.7 [v2:172.21.15.88:6824/2429727730,v1:172.21.15.88:6825/2429727730] boot 2024-09-16T09:47:39.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:38 smithi088 ceph-mon[52758]: osdmap e155: 8 total, 8 up, 8 in 2024-09-16T09:47:39.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:38 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-16T09:47:39.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:38 smithi088 ceph-mon[52758]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-16T09:47:39.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:38 smithi088 ceph-mon[52758]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-16T09:47:39.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:38 smithi088 ceph-mon[52758]: osdmap e156: 8 total, 8 up, 8 in 2024-09-16T09:47:39.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:38 smithi088 ceph-mon[52758]: Standby daemon mds.foofs.smithi088.dbunob assigned to filesystem foofs as rank 0 2024-09-16T09:47:39.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:38 smithi088 ceph-mon[52758]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-16T09:47:39.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:38 smithi088 ceph-mon[52758]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-16T09:47:39.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:38 smithi088 ceph-mon[52758]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-16T09:47:39.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:38 smithi088 ceph-mon[52758]: fsmap foofs:1/1 {0=foofs.smithi088.dbunob=up:replay} 2024-09-16T09:47:39.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:38 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi078[63905]: 2024-09-16T09:47:38.708+0000 7f478ef48640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-16T09:47:39.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:38 smithi078 ceph-mon[63909]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-16T09:47:39.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:38 smithi078 ceph-mon[63909]: osd.7 [v2:172.21.15.88:6824/2429727730,v1:172.21.15.88:6825/2429727730] boot 2024-09-16T09:47:39.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:38 smithi078 ceph-mon[63909]: osdmap e155: 8 total, 8 up, 8 in 2024-09-16T09:47:39.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:38 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-16T09:47:39.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:38 smithi078 ceph-mon[63909]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-16T09:47:39.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:38 smithi078 ceph-mon[63909]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-16T09:47:39.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:38 smithi078 ceph-mon[63909]: osdmap e156: 8 total, 8 up, 8 in 2024-09-16T09:47:39.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:38 smithi078 ceph-mon[63909]: Standby daemon mds.foofs.smithi088.dbunob assigned to filesystem foofs as rank 0 2024-09-16T09:47:39.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:38 smithi078 ceph-mon[63909]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-16T09:47:39.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:38 smithi078 ceph-mon[63909]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-16T09:47:39.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:38 smithi078 ceph-mon[63909]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-16T09:47:39.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:38 smithi078 ceph-mon[63909]: fsmap foofs:1/1 {0=foofs.smithi088.dbunob=up:replay} 2024-09-16T09:47:39.587 INFO:teuthology.orchestra.run.smithi078.stdout: 5 257651 9.14 MB/sec execute 599 sec latency 1308.941 ms 2024-09-16T09:47:40.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:39 smithi088 ceph-mon[52758]: pgmap v293: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 355 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.4 KiB/s rd, 5.5 MiB/s wr, 610 op/s; 260/1677 objects degraded (15.504%) 2024-09-16T09:47:40.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:39 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 260/1677 objects degraded (15.504%), 35 pgs degraded (PG_DEGRADED) 2024-09-16T09:47:40.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:39 smithi088 ceph-mon[52758]: osdmap e157: 8 total, 8 up, 8 in 2024-09-16T09:47:40.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:39 smithi078 ceph-mon[63909]: pgmap v293: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 355 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.4 KiB/s rd, 5.5 MiB/s wr, 610 op/s; 260/1677 objects degraded (15.504%) 2024-09-16T09:47:40.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:39 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 260/1677 objects degraded (15.504%), 35 pgs degraded (PG_DEGRADED) 2024-09-16T09:47:40.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:39 smithi078 ceph-mon[63909]: osdmap e157: 8 total, 8 up, 8 in 2024-09-16T09:47:40.551 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:47:40 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:47:40.163+0000 7fcc33cb0640 -1 bdev(0x55e1264eac00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 3 2024-09-16T09:47:40.551 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:47:40 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:47:40.164+0000 7fcc374b7640 -1 bdev(0x55e1264eac00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 4 2024-09-16T09:47:40.587 INFO:teuthology.orchestra.run.smithi078.stdout: 5 cleanup 600 sec 2024-09-16T09:47:41.587 INFO:teuthology.orchestra.run.smithi078.stdout: 5 cleanup 601 sec 2024-09-16T09:47:42.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:41 smithi078 ceph-mon[63909]: pgmap v297: 97 pgs: 6 remapped+peering, 4 peering, 2 active+undersized, 21 active+undersized+degraded, 64 active+clean; 350 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 3.3 MiB/s wr, 452 op/s; 152/1554 objects degraded (9.781%) 2024-09-16T09:47:42.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:41 smithi078 ceph-mon[63909]: osdmap e158: 8 total, 8 up, 8 in 2024-09-16T09:47:42.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:41 smithi088 ceph-mon[52758]: pgmap v297: 97 pgs: 6 remapped+peering, 4 peering, 2 active+undersized, 21 active+undersized+degraded, 64 active+clean; 350 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 3.3 MiB/s wr, 452 op/s; 152/1554 objects degraded (9.781%) 2024-09-16T09:47:42.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:41 smithi088 ceph-mon[52758]: osdmap e158: 8 total, 8 up, 8 in 2024-09-16T09:47:42.587 INFO:teuthology.orchestra.run.smithi078.stdout: 5 cleanup 602 sec 2024-09-16T09:47:42.945 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-16T09:47:42.946 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:42 smithi078 ceph-mon[63909]: osdmap e159: 8 total, 8 up, 8 in 2024-09-16T09:47:42.946 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:42 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:42.946 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:42 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:47:43.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:42 smithi088 ceph-mon[52758]: osdmap e159: 8 total, 8 up, 8 in 2024-09-16T09:47:43.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:42 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:43.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:42 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:47:43.587 INFO:teuthology.orchestra.run.smithi078.stdout: 5 cleanup 603 sec 2024-09-16T09:47:43.671 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T09:47:43.672 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (9m) 2m ago 20m 16.2M - 0.27.0 11f11916f8cd 3ebb6832fbcc 2024-09-16T09:47:43.672 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (5m) 2m ago 20m 6899k - 19.3.0-4882-g66e33d6a 261f2dc7b20f 72965213139c 2024-09-16T09:47:43.672 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi088 smithi088 running (5m) 16s ago 18m 6891k - 19.3.0-4882-g66e33d6a 261f2dc7b20f eac0e6855b1b 2024-09-16T09:47:43.672 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (9m) 2m ago 18m 45.2M - dad864ee21e9 4a91b97e29b1 2024-09-16T09:47:43.672 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.hiotnn smithi078 running (12m) 2m ago 12m 38.1M - 17.2.0 e1d6a67b021e c43db7767ab9 2024-09-16T09:47:43.672 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi088.dbunob smithi088 running (12m) 16s ago 12m 18.2M - 17.2.0 e1d6a67b021e d235f1bfbb05 2024-09-16T09:47:43.672 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rrfxge smithi078 *:8443,9283,8765 running (8m) 2m ago 21m 519M - 19.3.0-4882-g66e33d6a 261f2dc7b20f 0d25f2f30f1e 2024-09-16T09:47:43.672 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi088.ncxfdz smithi088 *:8443,9283,8765 running (7m) 16s ago 17m 457M - 19.3.0-4882-g66e33d6a 261f2dc7b20f e2d03f701acd 2024-09-16T09:47:43.672 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (7m) 2m ago 21m 58.2M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f f5a3a0fec757 2024-09-16T09:47:43.673 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi088 smithi088 running (6m) 16s ago 17m 48.7M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f 2e552147327a 2024-09-16T09:47:43.673 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.wawosh smithi078 *:2049 running (12m) 2m ago 12m 280M - 4.0 e1d6a67b021e 2544ed9d67f5 2024-09-16T09:47:43.673 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi088.bnccmw smithi088 *:2049 running (12m) 16s ago 12m 84.3M - 4.0 e1d6a67b021e e606adb67049 2024-09-16T09:47:43.673 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (8m) 2m ago 18m 9.82M - 1.7.0 72c9c2088986 895ac3ccb2f3 2024-09-16T09:47:43.673 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi088 smithi088 *:9100 running (8m) 16s ago 17m 11.7M - 1.7.0 72c9c2088986 c84a2e50de53 2024-09-16T09:47:43.673 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (5m) 2m ago 17m 348M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 85008b499cb7 2024-09-16T09:47:43.673 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (4m) 2m ago 16m 271M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 12c137fcba70 2024-09-16T09:47:43.673 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (3m) 2m ago 16m 211M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 23a8b69c5c64 2024-09-16T09:47:43.673 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (2m) 2m ago 15m 11.2M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 00c9aa84d41d 2024-09-16T09:47:43.673 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi088 running (2m) 16s ago 15m 318M 2704M 19.3.0-4882-g66e33d6a 261f2dc7b20f 1d92e66b75d9 2024-09-16T09:47:43.673 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi088 running (93s) 16s ago 14m 287M 2704M 19.3.0-4882-g66e33d6a 261f2dc7b20f 43abc1197fe2 2024-09-16T09:47:43.674 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi088 running (57s) 16s ago 14m 244M 2704M 19.3.0-4882-g66e33d6a 261f2dc7b20f 31302f8f7d8f 2024-09-16T09:47:43.674 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi088 running (21s) 16s ago 14m 11.1M 2704M 19.3.0-4882-g66e33d6a 261f2dc7b20f f3768fef8040 2024-09-16T09:47:43.674 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (7m) 2m ago 17m 53.1M - 2.51.0 1d3b7f56885b 4c956ead9125 2024-09-16T09:47:44.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:43 smithi078 ceph-mon[63909]: pgmap v300: 97 pgs: 1 active+undersized+remapped, 4 remapped+peering, 5 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized, 4 active+undersized+degraded, 80 active+clean; 349 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 2.0 MiB/s wr, 162 op/s; 28/1530 objects degraded (1.830%); 3.3 MiB/s, 0 keys/s, 48 objects/s recovering 2024-09-16T09:47:44.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:43 smithi078 ceph-mon[63909]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-16T09:47:44.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:43 smithi078 ceph-mon[63909]: reconnect by client.24337 172.21.15.88:0/3730125002 after 0 2024-09-16T09:47:44.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:43 smithi078 ceph-mon[63909]: reconnect by client.14598 172.21.15.78:0/1110678352 after 0.000999981 2024-09-16T09:47:44.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:43 smithi078 ceph-mon[63909]: mds.? [v2:172.21.15.88:6832/861752956,v1:172.21.15.88:6833/861752956] up:reconnect 2024-09-16T09:47:44.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:43 smithi078 ceph-mon[63909]: mds.? [v2:172.21.15.78:6834/2976710836,v1:172.21.15.78:6835/2976710836] up:boot 2024-09-16T09:47:44.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:43 smithi078 ceph-mon[63909]: fsmap foofs:1/1 {0=foofs.smithi088.dbunob=up:reconnect} 1 up:standby 2024-09-16T09:47:44.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:43 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "mds metadata", "who": "foofs.smithi078.hiotnn"} : dispatch 2024-09-16T09:47:44.098 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:47:44.098 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-16T09:47:44.098 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:47:44.098 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:47:44.098 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-16T09:47:44.099 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:47:44.099 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:47:44.099 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-16T09:47:44.099 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 8 2024-09-16T09:47:44.099 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:47:44.099 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-16T09:47:44.099 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T09:47:44.099 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:47:44.099 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-16T09:47:44.099 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-16T09:47:44.099 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 12 2024-09-16T09:47:44.099 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-16T09:47:44.099 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:47:44.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:43 smithi088 ceph-mon[52758]: pgmap v300: 97 pgs: 1 active+undersized+remapped, 4 remapped+peering, 5 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized, 4 active+undersized+degraded, 80 active+clean; 349 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 2.0 MiB/s wr, 162 op/s; 28/1530 objects degraded (1.830%); 3.3 MiB/s, 0 keys/s, 48 objects/s recovering 2024-09-16T09:47:44.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:43 smithi088 ceph-mon[52758]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-16T09:47:44.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:43 smithi088 ceph-mon[52758]: reconnect by client.24337 172.21.15.88:0/3730125002 after 0 2024-09-16T09:47:44.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:43 smithi088 ceph-mon[52758]: reconnect by client.14598 172.21.15.78:0/1110678352 after 0.000999981 2024-09-16T09:47:44.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:43 smithi088 ceph-mon[52758]: mds.? [v2:172.21.15.88:6832/861752956,v1:172.21.15.88:6833/861752956] up:reconnect 2024-09-16T09:47:44.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:43 smithi088 ceph-mon[52758]: mds.? [v2:172.21.15.78:6834/2976710836,v1:172.21.15.78:6835/2976710836] up:boot 2024-09-16T09:47:44.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:43 smithi088 ceph-mon[52758]: fsmap foofs:1/1 {0=foofs.smithi088.dbunob=up:reconnect} 1 up:standby 2024-09-16T09:47:44.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:43 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "mds metadata", "who": "foofs.smithi078.hiotnn"} : dispatch 2024-09-16T09:47:44.466 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:47:44.467 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f", 2024-09-16T09:47:44.467 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-16T09:47:44.467 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T09:47:44.467 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-16T09:47:44.467 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-09-16T09:47:44.467 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-16T09:47:44.467 INFO:teuthology.orchestra.run.smithi078.stdout: "osd", 2024-09-16T09:47:44.467 INFO:teuthology.orchestra.run.smithi078.stdout: "mon" 2024-09-16T09:47:44.467 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-16T09:47:44.467 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "14/23 daemons upgraded", 2024-09-16T09:47:44.467 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading mds daemons", 2024-09-16T09:47:44.467 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-16T09:47:44.467 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:47:44.587 INFO:teuthology.orchestra.run.smithi078.stdout: 5 cleanup 604 sec 2024-09-16T09:47:44.922 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_WARN Degraded data redundancy: 1 pg degraded 2024-09-16T09:47:44.923 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 1 pg degraded 2024-09-16T09:47:44.923 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.9 is active+recovering+undersized+degraded+remapped, acting [4,2] 2024-09-16T09:47:45.086 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:44 smithi078 ceph-mon[63909]: from='client.34436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:47:45.086 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:44 smithi078 ceph-mon[63909]: from='client.34444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:47:45.086 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:44 smithi078 ceph-mon[63909]: from='client.34448 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:47:45.086 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:44 smithi078 ceph-mon[63909]: mds.? [v2:172.21.15.88:6832/861752956,v1:172.21.15.88:6833/861752956] up:rejoin 2024-09-16T09:47:45.086 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:44 smithi078 ceph-mon[63909]: fsmap foofs:1/1 {0=foofs.smithi088.dbunob=up:rejoin} 1 up:standby 2024-09-16T09:47:45.086 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:44 smithi078 ceph-mon[63909]: daemon mds.foofs.smithi088.dbunob is now active in filesystem foofs as rank 0 2024-09-16T09:47:45.086 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:44 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/3436391033' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:47:45.087 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:44 smithi078 ceph-mon[63909]: Health check update: Degraded data redundancy: 28/1530 objects degraded (1.830%), 5 pgs degraded (PG_DEGRADED) 2024-09-16T09:47:45.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:44 smithi088 ceph-mon[52758]: from='client.34436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:47:45.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:44 smithi088 ceph-mon[52758]: from='client.34444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:47:45.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:44 smithi088 ceph-mon[52758]: from='client.34448 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:47:45.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:44 smithi088 ceph-mon[52758]: mds.? [v2:172.21.15.88:6832/861752956,v1:172.21.15.88:6833/861752956] up:rejoin 2024-09-16T09:47:45.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:44 smithi088 ceph-mon[52758]: fsmap foofs:1/1 {0=foofs.smithi088.dbunob=up:rejoin} 1 up:standby 2024-09-16T09:47:45.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:44 smithi088 ceph-mon[52758]: daemon mds.foofs.smithi088.dbunob is now active in filesystem foofs as rank 0 2024-09-16T09:47:45.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:44 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/3436391033' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:47:45.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:44 smithi088 ceph-mon[52758]: Health check update: Degraded data redundancy: 28/1530 objects degraded (1.830%), 5 pgs degraded (PG_DEGRADED) 2024-09-16T09:47:45.587 INFO:teuthology.orchestra.run.smithi078.stdout: 5 cleanup 605 sec 2024-09-16T09:47:46.013 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:45 smithi078 ceph-mon[63909]: pgmap v301: 97 pgs: 1 active+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 94 active+clean; 350 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 37 MiB/s rd, 1.4 MiB/s wr, 132 op/s; 3.3 MiB/s, 0 keys/s, 58 objects/s recovering 2024-09-16T09:47:46.013 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:45 smithi078 ceph-mon[63909]: from='client.34456 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:47:46.014 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:45 smithi078 ceph-mon[63909]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-16T09:47:46.014 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:45 smithi078 ceph-mon[63909]: mds.? [v2:172.21.15.88:6832/861752956,v1:172.21.15.88:6833/861752956] up:active 2024-09-16T09:47:46.014 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:45 smithi078 ceph-mon[63909]: fsmap foofs:1 {0=foofs.smithi088.dbunob=up:active} 1 up:standby 2024-09-16T09:47:46.014 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:45 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/250170394' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:47:46.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:45 smithi088 ceph-mon[52758]: pgmap v301: 97 pgs: 1 active+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 94 active+clean; 350 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 37 MiB/s rd, 1.4 MiB/s wr, 132 op/s; 3.3 MiB/s, 0 keys/s, 58 objects/s recovering 2024-09-16T09:47:46.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:45 smithi088 ceph-mon[52758]: from='client.34456 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:47:46.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:45 smithi088 ceph-mon[52758]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-16T09:47:46.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:45 smithi088 ceph-mon[52758]: mds.? [v2:172.21.15.88:6832/861752956,v1:172.21.15.88:6833/861752956] up:active 2024-09-16T09:47:46.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:45 smithi088 ceph-mon[52758]: fsmap foofs:1 {0=foofs.smithi088.dbunob=up:active} 1 up:standby 2024-09-16T09:47:46.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:45 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/250170394' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:47:46.428 INFO:teuthology.orchestra.run.smithi078.stdout: 0 cleanup 606 sec 2024-09-16T09:47:46.428 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-16T09:47:46.428 INFO:teuthology.orchestra.run.smithi078.stdout: Operation Count AvgLat MaxLat 2024-09-16T09:47:46.428 INFO:teuthology.orchestra.run.smithi078.stdout: ---------------------------------------- 2024-09-16T09:47:46.428 INFO:teuthology.orchestra.run.smithi078.stdout: NTCreateX 175680 4.746 1081.731 2024-09-16T09:47:46.428 INFO:teuthology.orchestra.run.smithi078.stdout: Close 129015 0.129 15.195 2024-09-16T09:47:46.428 INFO:teuthology.orchestra.run.smithi078.stdout: Rename 7449 10.170 1056.485 2024-09-16T09:47:46.428 INFO:teuthology.orchestra.run.smithi078.stdout: Unlink 35488 7.180 1036.171 2024-09-16T09:47:46.428 INFO:teuthology.orchestra.run.smithi078.stdout: Qpathinfo 159322 1.210 116.989 2024-09-16T09:47:46.429 INFO:teuthology.orchestra.run.smithi078.stdout: Qfileinfo 27780 0.006 1.154 2024-09-16T09:47:46.429 INFO:teuthology.orchestra.run.smithi078.stdout: Qfsinfo 29207 0.596 51.457 2024-09-16T09:47:46.429 INFO:teuthology.orchestra.run.smithi078.stdout: Sfileinfo 14343 14.646 6149.715 2024-09-16T09:47:46.429 INFO:teuthology.orchestra.run.smithi078.stdout: Find 61565 0.907 23.995 2024-09-16T09:47:46.429 INFO:teuthology.orchestra.run.smithi078.stdout: WriteX 86806 15.653 4838.275 2024-09-16T09:47:46.429 INFO:teuthology.orchestra.run.smithi078.stdout: ReadX 275649 0.010 10.496 2024-09-16T09:47:46.429 INFO:teuthology.orchestra.run.smithi078.stdout: LockX 572 0.350 2.531 2024-09-16T09:47:46.429 INFO:teuthology.orchestra.run.smithi078.stdout: UnlockX 572 0.306 1.909 2024-09-16T09:47:46.429 INFO:teuthology.orchestra.run.smithi078.stdout: Flush 12305 0.002 0.031 2024-09-16T09:47:46.429 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-16T09:47:46.429 INFO:teuthology.orchestra.run.smithi078.stdout:Throughput 9.14382 MB/sec 5 clients 5 procs max_latency=2122.998 ms 2024-09-16T09:47:46.431 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-09-16T09:47:46.467 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-09-16T09:47:46.585 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-09-16T09:47:46.628 INFO:teuthology.orchestra.run.smithi078.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-16T09:47:46.628 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-16T09:47:46.628 INFO:teuthology.orchestra.run.smithi078.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-09-16T09:47:46.629 INFO:teuthology.orchestra.run.smithi078.stdout:3 of 5 processes prepared for launch 0 sec 2024-09-16T09:47:46.729 INFO:teuthology.orchestra.run.smithi078.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-16T09:47:46.729 INFO:teuthology.orchestra.run.smithi078.stdout:releasing clients 2024-09-16T09:47:47.121 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:46 smithi078 ceph-mon[63909]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg degraded) 2024-09-16T09:47:47.121 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:46 smithi078 ceph-mon[63909]: Cluster is now healthy 2024-09-16T09:47:47.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:46 smithi088 ceph-mon[52758]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg degraded) 2024-09-16T09:47:47.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:46 smithi088 ceph-mon[52758]: Cluster is now healthy 2024-09-16T09:47:47.971 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:47 smithi078 ceph-mon[63909]: pgmap v302: 97 pgs: 1 peering, 96 active+clean; 348 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 34 MiB/s rd, 429 KiB/s wr, 70 op/s; 2.8 MiB/s, 0 keys/s, 49 objects/s recovering 2024-09-16T09:47:48.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:47 smithi088 ceph-mon[52758]: pgmap v302: 97 pgs: 1 peering, 96 active+clean; 348 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 34 MiB/s rd, 429 KiB/s wr, 70 op/s; 2.8 MiB/s, 0 keys/s, 49 objects/s recovering 2024-09-16T09:47:48.729 INFO:teuthology.orchestra.run.smithi078.stdout: 5 247 12.22 MB/sec execute 1 sec latency 181.498 ms 2024-09-16T09:47:49.729 INFO:teuthology.orchestra.run.smithi078.stdout: 5 346 13.65 MB/sec execute 2 sec latency 198.499 ms 2024-09-16T09:47:49.892 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:49 smithi078 ceph-mon[63909]: pgmap v303: 97 pgs: 97 active+clean; 345 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 1.2 MiB/s wr, 115 op/s; 2.2 MiB/s, 0 keys/s, 39 objects/s recovering 2024-09-16T09:47:50.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:49 smithi088 ceph-mon[52758]: pgmap v303: 97 pgs: 97 active+clean; 345 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 1.2 MiB/s wr, 115 op/s; 2.2 MiB/s, 0 keys/s, 39 objects/s recovering 2024-09-16T09:47:50.729 INFO:teuthology.orchestra.run.smithi078.stdout: 5 418 13.00 MB/sec execute 3 sec latency 132.709 ms 2024-09-16T09:47:51.729 INFO:teuthology.orchestra.run.smithi078.stdout: 5 508 13.35 MB/sec execute 4 sec latency 118.756 ms 2024-09-16T09:47:52.024 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:51 smithi078 ceph-mon[63909]: pgmap v304: 97 pgs: 97 active+clean; 321 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 3.0 MiB/s wr, 120 op/s; 1.3 MiB/s, 0 keys/s, 26 objects/s recovering 2024-09-16T09:47:52.024 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:51 smithi078 ceph-mon[63909]: daemon mds.foofs.smithi078.hiotnn restarted 2024-09-16T09:47:52.024 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:51 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:52.024 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:51 smithi078 ceph-mon[63909]: osdmap e160: 8 total, 8 up, 8 in 2024-09-16T09:47:52.024 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:51 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:52.024 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:51 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:47:52.024 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:51 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:52.024 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:51 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:52.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:51 smithi088 ceph-mon[52758]: pgmap v304: 97 pgs: 97 active+clean; 321 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 3.0 MiB/s wr, 120 op/s; 1.3 MiB/s, 0 keys/s, 26 objects/s recovering 2024-09-16T09:47:52.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:51 smithi088 ceph-mon[52758]: daemon mds.foofs.smithi078.hiotnn restarted 2024-09-16T09:47:52.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:51 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:52.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:51 smithi088 ceph-mon[52758]: osdmap e160: 8 total, 8 up, 8 in 2024-09-16T09:47:52.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:51 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:52.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:51 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:47:52.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:51 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:52.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:51 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:52.729 INFO:teuthology.orchestra.run.smithi078.stdout: 5 cleanup 5 sec 2024-09-16T09:47:52.917 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:52 smithi078 ceph-mon[63909]: mds.? [v2:172.21.15.78:6834/487538732,v1:172.21.15.78:6835/487538732] up:boot 2024-09-16T09:47:52.917 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:52 smithi078 ceph-mon[63909]: fsmap foofs:1 {0=foofs.smithi088.dbunob=up:active} 1 up:standby 2024-09-16T09:47:52.918 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:52 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "mds metadata", "who": "foofs.smithi078.hiotnn"} : dispatch 2024-09-16T09:47:52.918 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:52 smithi078 ceph-mon[63909]: pgmap v306: 97 pgs: 97 active+clean; 307 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 5.8 MiB/s wr, 131 op/s; 526 KiB/s, 13 objects/s recovering 2024-09-16T09:47:53.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:52 smithi088 ceph-mon[52758]: mds.? [v2:172.21.15.78:6834/487538732,v1:172.21.15.78:6835/487538732] up:boot 2024-09-16T09:47:53.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:52 smithi088 ceph-mon[52758]: fsmap foofs:1 {0=foofs.smithi088.dbunob=up:active} 1 up:standby 2024-09-16T09:47:53.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:52 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "mds metadata", "who": "foofs.smithi078.hiotnn"} : dispatch 2024-09-16T09:47:53.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:52 smithi088 ceph-mon[52758]: pgmap v306: 97 pgs: 97 active+clean; 307 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 5.8 MiB/s wr, 131 op/s; 526 KiB/s, 13 objects/s recovering 2024-09-16T09:47:53.532 INFO:teuthology.orchestra.run.smithi078.stdout: 0 cleanup 6 sec 2024-09-16T09:47:53.532 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-16T09:47:53.532 INFO:teuthology.orchestra.run.smithi078.stdout: Operation Count AvgLat MaxLat 2024-09-16T09:47:53.532 INFO:teuthology.orchestra.run.smithi078.stdout: ---------------------------------------- 2024-09-16T09:47:53.532 INFO:teuthology.orchestra.run.smithi078.stdout: NTCreateX 335 40.132 198.497 2024-09-16T09:47:53.532 INFO:teuthology.orchestra.run.smithi078.stdout: Close 318 14.869 103.036 2024-09-16T09:47:53.532 INFO:teuthology.orchestra.run.smithi078.stdout: Rename 2 43.099 43.145 2024-09-16T09:47:53.533 INFO:teuthology.orchestra.run.smithi078.stdout: Unlink 7 12.228 20.596 2024-09-16T09:47:53.533 INFO:teuthology.orchestra.run.smithi078.stdout: Qpathinfo 31 4.160 42.432 2024-09-16T09:47:53.533 INFO:teuthology.orchestra.run.smithi078.stdout: Qfileinfo 301 18.496 112.970 2024-09-16T09:47:53.533 INFO:teuthology.orchestra.run.smithi078.stdout: Qfsinfo 45 0.381 0.673 2024-09-16T09:47:53.533 INFO:teuthology.orchestra.run.smithi078.stdout: Sfileinfo 20 17.624 32.384 2024-09-16T09:47:53.533 INFO:teuthology.orchestra.run.smithi078.stdout: Find 40 0.846 1.646 2024-09-16T09:47:53.533 INFO:teuthology.orchestra.run.smithi078.stdout: WriteX 1433 0.027 0.083 2024-09-16T09:47:53.533 INFO:teuthology.orchestra.run.smithi078.stdout: ReadX 142 0.016 0.352 2024-09-16T09:47:53.533 INFO:teuthology.orchestra.run.smithi078.stdout: Flush 38 17.428 31.136 2024-09-16T09:47:53.533 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-16T09:47:53.533 INFO:teuthology.orchestra.run.smithi078.stdout:Throughput 13.3452 MB/sec 5 clients 5 procs max_latency=198.499 ms 2024-09-16T09:47:53.534 DEBUG:teuthology.parallel:result is None 2024-09-16T09:47:55.392 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:55 smithi078 ceph-mon[63909]: pgmap v307: 97 pgs: 97 active+clean; 331 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 9.7 MiB/s wr, 254 op/s; 0 B/s, 0 objects/s recovering 2024-09-16T09:47:55.547 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:55 smithi088 ceph-mon[52758]: pgmap v307: 97 pgs: 97 active+clean; 331 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 9.7 MiB/s wr, 254 op/s; 0 B/s, 0 objects/s recovering 2024-09-16T09:47:57.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:57 smithi078 ceph-mon[63909]: pgmap v308: 97 pgs: 97 active+clean; 333 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 450 KiB/s rd, 10 MiB/s wr, 258 op/s 2024-09-16T09:47:57.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:57 smithi088 ceph-mon[52758]: pgmap v308: 97 pgs: 97 active+clean; 333 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 450 KiB/s rd, 10 MiB/s wr, 258 op/s 2024-09-16T09:47:58.603 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:58 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:58.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:58 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:47:58.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:58 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:58.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:58 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:58.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:58 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:58.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:58 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:58.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:58 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:58.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:58 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:58.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:58 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:47:58.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:58 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:58.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:58 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:58.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:58 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:58.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:58 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:58.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:58 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:47:59.625 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:47:59 smithi078 ceph-mon[63909]: pgmap v309: 97 pgs: 97 active+clean; 297 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 10 MiB/s wr, 225 op/s 2024-09-16T09:47:59.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:47:59 smithi088 ceph-mon[52758]: pgmap v309: 97 pgs: 97 active+clean; 297 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 10 MiB/s wr, 225 op/s 2024-09-16T09:48:01.548 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:01 smithi078 ceph-mon[63909]: pgmap v310: 97 pgs: 97 active+clean; 259 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 8.3 MiB/s wr, 206 op/s 2024-09-16T09:48:01.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:01 smithi088 ceph-mon[52758]: pgmap v310: 97 pgs: 97 active+clean; 259 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 8.3 MiB/s wr, 206 op/s 2024-09-16T09:48:02.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:02 smithi088 ceph-mon[52758]: Detected new or changed devices on smithi078 2024-09-16T09:48:02.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:02 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:02.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:02 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:02.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:02 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:48:02.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:02 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:48:02.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:02 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:02.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:02 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:02.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:02 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:48:02.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:02 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:02.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:02 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:02.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:02 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:02.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:02 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:02.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:02 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi088.dbunob"]} : dispatch 2024-09-16T09:48:03.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:02 smithi078 ceph-mon[63909]: Detected new or changed devices on smithi078 2024-09-16T09:48:03.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:02 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:03.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:02 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:03.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:02 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:48:03.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:02 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:48:03.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:02 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:03.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:02 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:03.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:02 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:48:03.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:02 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:03.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:02 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:03.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:02 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:03.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:02 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:03.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:02 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi088.dbunob"]} : dispatch 2024-09-16T09:48:03.820 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:03 smithi088 ceph-mon[52758]: Upgrade: It appears safe to stop mds.foofs.smithi088.dbunob 2024-09-16T09:48:03.820 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:03 smithi088 ceph-mon[52758]: pgmap v311: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.3 KiB/s rd, 6.6 MiB/s wr, 183 op/s 2024-09-16T09:48:03.820 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:03 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:03.820 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:03 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi088.dbunob", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-16T09:48:03.820 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:03 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:48:04.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:03 smithi078 ceph-mon[63909]: Upgrade: It appears safe to stop mds.foofs.smithi088.dbunob 2024-09-16T09:48:04.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:03 smithi078 ceph-mon[63909]: pgmap v311: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.3 KiB/s rd, 6.6 MiB/s wr, 183 op/s 2024-09-16T09:48:04.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:03 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:04.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:03 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi088.dbunob", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-16T09:48:04.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:03 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:48:04.918 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:04 smithi088 ceph-mon[52758]: Upgrade: Updating mds.foofs.smithi088.dbunob 2024-09-16T09:48:04.918 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:04 smithi088 ceph-mon[52758]: Deploying daemon mds.foofs.smithi088.dbunob on smithi088 2024-09-16T09:48:05.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:04 smithi078 ceph-mon[63909]: Upgrade: Updating mds.foofs.smithi088.dbunob 2024-09-16T09:48:05.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:04 smithi078 ceph-mon[63909]: Deploying daemon mds.foofs.smithi088.dbunob on smithi088 2024-09-16T09:48:05.551 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:48:05 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:48:05.244+0000 7fcc334af640 -1 bdev(0x55e1264eac00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 6 2024-09-16T09:48:05.551 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:48:05 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:48:05.244+0000 7fcc33cb0640 -1 bdev(0x55e1264eac00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 6 2024-09-16T09:48:05.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:05 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi078[63905]: 2024-09-16T09:48:05.161+0000 7f478ef48640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-16T09:48:06.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:05 smithi088 ceph-mon[52758]: pgmap v312: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.3 MiB/s wr, 149 op/s 2024-09-16T09:48:06.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:05 smithi088 ceph-mon[52758]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-16T09:48:06.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:05 smithi088 ceph-mon[52758]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-16T09:48:06.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:05 smithi088 ceph-mon[52758]: osdmap e161: 8 total, 8 up, 8 in 2024-09-16T09:48:06.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:05 smithi088 ceph-mon[52758]: Standby daemon mds.foofs.smithi078.hiotnn assigned to filesystem foofs as rank 0 2024-09-16T09:48:06.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:05 smithi088 ceph-mon[52758]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-16T09:48:06.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:05 smithi088 ceph-mon[52758]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-16T09:48:06.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:05 smithi088 ceph-mon[52758]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-16T09:48:06.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:05 smithi088 ceph-mon[52758]: fsmap foofs:1/1 {0=foofs.smithi078.hiotnn=up:replay} 2024-09-16T09:48:06.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:05 smithi078 ceph-mon[63909]: pgmap v312: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.3 MiB/s wr, 149 op/s 2024-09-16T09:48:06.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:05 smithi078 ceph-mon[63909]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-16T09:48:06.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:05 smithi078 ceph-mon[63909]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-16T09:48:06.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:05 smithi078 ceph-mon[63909]: osdmap e161: 8 total, 8 up, 8 in 2024-09-16T09:48:06.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:05 smithi078 ceph-mon[63909]: Standby daemon mds.foofs.smithi078.hiotnn assigned to filesystem foofs as rank 0 2024-09-16T09:48:06.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:05 smithi078 ceph-mon[63909]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-16T09:48:06.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:05 smithi078 ceph-mon[63909]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-16T09:48:06.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:05 smithi078 ceph-mon[63909]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-16T09:48:06.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:05 smithi078 ceph-mon[63909]: fsmap foofs:1/1 {0=foofs.smithi078.hiotnn=up:replay} 2024-09-16T09:48:08.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:07 smithi088 ceph-mon[52758]: pgmap v314: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 514 KiB/s wr, 34 op/s 2024-09-16T09:48:08.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:07 smithi088 ceph-mon[52758]: reconnect by client.14598 172.21.15.78:0/1110678352 after 0.000999985 2024-09-16T09:48:08.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:07 smithi088 ceph-mon[52758]: mds.? [v2:172.21.15.78:6834/487538732,v1:172.21.15.78:6835/487538732] up:reconnect 2024-09-16T09:48:08.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:07 smithi088 ceph-mon[52758]: fsmap foofs:1/1 {0=foofs.smithi078.hiotnn=up:reconnect} 2024-09-16T09:48:08.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:07 smithi088 ceph-mon[52758]: reconnect by client.24337 172.21.15.88:0/3730125002 after 0.0469993 2024-09-16T09:48:08.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:07 smithi078 ceph-mon[63909]: pgmap v314: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 514 KiB/s wr, 34 op/s 2024-09-16T09:48:08.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:07 smithi078 ceph-mon[63909]: reconnect by client.14598 172.21.15.78:0/1110678352 after 0.000999985 2024-09-16T09:48:08.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:07 smithi078 ceph-mon[63909]: mds.? [v2:172.21.15.78:6834/487538732,v1:172.21.15.78:6835/487538732] up:reconnect 2024-09-16T09:48:08.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:07 smithi078 ceph-mon[63909]: fsmap foofs:1/1 {0=foofs.smithi078.hiotnn=up:reconnect} 2024-09-16T09:48:08.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:07 smithi078 ceph-mon[63909]: reconnect by client.24337 172.21.15.88:0/3730125002 after 0.0469993 2024-09-16T09:48:09.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:08 smithi088 ceph-mon[52758]: mds.? [v2:172.21.15.78:6834/487538732,v1:172.21.15.78:6835/487538732] up:rejoin 2024-09-16T09:48:09.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:08 smithi088 ceph-mon[52758]: fsmap foofs:1/1 {0=foofs.smithi078.hiotnn=up:rejoin} 2024-09-16T09:48:09.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:08 smithi088 ceph-mon[52758]: daemon mds.foofs.smithi078.hiotnn is now active in filesystem foofs as rank 0 2024-09-16T09:48:09.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:08 smithi088 ceph-mon[52758]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-16T09:48:09.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:08 smithi088 ceph-mon[52758]: mds.? [v2:172.21.15.78:6834/487538732,v1:172.21.15.78:6835/487538732] up:active 2024-09-16T09:48:09.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:08 smithi088 ceph-mon[52758]: fsmap foofs:1 {0=foofs.smithi078.hiotnn=up:active} 2024-09-16T09:48:09.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:08 smithi078 ceph-mon[63909]: mds.? [v2:172.21.15.78:6834/487538732,v1:172.21.15.78:6835/487538732] up:rejoin 2024-09-16T09:48:09.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:08 smithi078 ceph-mon[63909]: fsmap foofs:1/1 {0=foofs.smithi078.hiotnn=up:rejoin} 2024-09-16T09:48:09.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:08 smithi078 ceph-mon[63909]: daemon mds.foofs.smithi078.hiotnn is now active in filesystem foofs as rank 0 2024-09-16T09:48:09.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:08 smithi078 ceph-mon[63909]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-16T09:48:09.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:08 smithi078 ceph-mon[63909]: mds.? [v2:172.21.15.78:6834/487538732,v1:172.21.15.78:6835/487538732] up:active 2024-09-16T09:48:09.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:08 smithi078 ceph-mon[63909]: fsmap foofs:1 {0=foofs.smithi078.hiotnn=up:active} 2024-09-16T09:48:09.932 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:09 smithi088 ceph-mon[52758]: pgmap v315: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 24 KiB/s wr, 23 op/s 2024-09-16T09:48:10.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:09 smithi078 ceph-mon[63909]: pgmap v315: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 24 KiB/s wr, 23 op/s 2024-09-16T09:48:11.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:11 smithi078 ceph-mon[63909]: pgmap v316: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 716 B/s wr, 14 op/s 2024-09-16T09:48:11.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:11 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:11.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:11 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:11.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:11 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:48:12.038 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:11 smithi088 ceph-mon[52758]: pgmap v316: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 716 B/s wr, 14 op/s 2024-09-16T09:48:12.038 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:11 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:12.038 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:11 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:12.038 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:11 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:48:12.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:12 smithi088 ceph-mon[52758]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-16T09:48:12.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:12 smithi088 ceph-mon[52758]: Cluster is now healthy 2024-09-16T09:48:12.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:12 smithi088 ceph-mon[52758]: mds.? [v2:172.21.15.88:6832/1163566373,v1:172.21.15.88:6833/1163566373] up:boot 2024-09-16T09:48:12.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:12 smithi088 ceph-mon[52758]: fsmap foofs:1 {0=foofs.smithi078.hiotnn=up:active} 1 up:standby 2024-09-16T09:48:12.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:12 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "mds metadata", "who": "foofs.smithi088.dbunob"} : dispatch 2024-09-16T09:48:12.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:12 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:48:13.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:12 smithi078 ceph-mon[63909]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-16T09:48:13.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:12 smithi078 ceph-mon[63909]: Cluster is now healthy 2024-09-16T09:48:13.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:12 smithi078 ceph-mon[63909]: mds.? [v2:172.21.15.88:6832/1163566373,v1:172.21.15.88:6833/1163566373] up:boot 2024-09-16T09:48:13.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:12 smithi078 ceph-mon[63909]: fsmap foofs:1 {0=foofs.smithi078.hiotnn=up:active} 1 up:standby 2024-09-16T09:48:13.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:12 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "mds metadata", "who": "foofs.smithi088.dbunob"} : dispatch 2024-09-16T09:48:13.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:12 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:48:13.924 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:13 smithi088 ceph-mon[52758]: pgmap v317: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 511 B/s wr, 11 op/s 2024-09-16T09:48:14.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:13 smithi078 ceph-mon[63909]: pgmap v317: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 511 B/s wr, 11 op/s 2024-09-16T09:48:15.293 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-16T09:48:15.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:15 smithi088 ceph-mon[52758]: pgmap v318: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 8.7 KiB/s wr, 15 op/s 2024-09-16T09:48:15.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:15 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:15.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:15 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:15.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:15 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:15.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:15 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:15.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:15 smithi078 ceph-mon[63909]: pgmap v318: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 8.7 KiB/s wr, 15 op/s 2024-09-16T09:48:15.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:15 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:15.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:15 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:15.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:15 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:15.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:15 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:15.992 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T09:48:15.992 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (9m) 18s ago 20m 18.8M - 0.27.0 11f11916f8cd 3ebb6832fbcc 2024-09-16T09:48:15.992 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (6m) 18s ago 20m 6899k - 19.3.0-4882-g66e33d6a 261f2dc7b20f 72965213139c 2024-09-16T09:48:15.992 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi088 smithi088 running (6m) 1s ago 18m 6891k - 19.3.0-4882-g66e33d6a 261f2dc7b20f eac0e6855b1b 2024-09-16T09:48:15.993 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (9m) 18s ago 18m 46.9M - dad864ee21e9 4a91b97e29b1 2024-09-16T09:48:15.993 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.hiotnn smithi078 running (25s) 18s ago 13m 11.8M - 19.3.0-4882-g66e33d6a 261f2dc7b20f e3c24b03534b 2024-09-16T09:48:15.993 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi088.dbunob smithi088 running (5s) 1s ago 13m 14.6M - 19.3.0-4882-g66e33d6a 261f2dc7b20f 8ffda444de95 2024-09-16T09:48:15.993 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rrfxge smithi078 *:8443,9283,8765 running (8m) 18s ago 21m 523M - 19.3.0-4882-g66e33d6a 261f2dc7b20f 0d25f2f30f1e 2024-09-16T09:48:15.993 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi088.ncxfdz smithi088 *:8443,9283,8765 running (7m) 1s ago 18m 458M - 19.3.0-4882-g66e33d6a 261f2dc7b20f e2d03f701acd 2024-09-16T09:48:15.993 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (7m) 18s ago 22m 66.3M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f f5a3a0fec757 2024-09-16T09:48:15.993 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi088 smithi088 running (7m) 1s ago 18m 50.7M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f 2e552147327a 2024-09-16T09:48:15.993 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.wawosh smithi078 *:2049 running (12m) 18s ago 12m 337M - 4.0 e1d6a67b021e 2544ed9d67f5 2024-09-16T09:48:15.993 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi088.bnccmw smithi088 *:2049 running (12m) 1s ago 12m 84.4M - 4.0 e1d6a67b021e e606adb67049 2024-09-16T09:48:15.993 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (9m) 18s ago 18m 10.2M - 1.7.0 72c9c2088986 895ac3ccb2f3 2024-09-16T09:48:15.993 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi088 smithi088 *:9100 running (8m) 1s ago 18m 11.7M - 1.7.0 72c9c2088986 c84a2e50de53 2024-09-16T09:48:15.993 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (5m) 18s ago 17m 398M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 85008b499cb7 2024-09-16T09:48:15.993 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (5m) 18s ago 17m 401M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 12c137fcba70 2024-09-16T09:48:15.994 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (4m) 18s ago 16m 385M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 23a8b69c5c64 2024-09-16T09:48:15.994 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (3m) 18s ago 16m 375M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 00c9aa84d41d 2024-09-16T09:48:15.994 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi088 running (2m) 1s ago 15m 343M 2704M 19.3.0-4882-g66e33d6a 261f2dc7b20f 1d92e66b75d9 2024-09-16T09:48:15.994 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi088 running (2m) 1s ago 15m 328M 2704M 19.3.0-4882-g66e33d6a 261f2dc7b20f 43abc1197fe2 2024-09-16T09:48:15.994 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi088 running (90s) 1s ago 15m 276M 2704M 19.3.0-4882-g66e33d6a 261f2dc7b20f 31302f8f7d8f 2024-09-16T09:48:15.994 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi088 running (53s) 1s ago 14m 288M 2704M 19.3.0-4882-g66e33d6a 261f2dc7b20f f3768fef8040 2024-09-16T09:48:15.994 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (8m) 18s ago 18m 55.5M - 2.51.0 1d3b7f56885b 4c956ead9125 2024-09-16T09:48:16.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:16 smithi078 ceph-mon[63909]: from='client.34472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:48:16.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:16 smithi078 ceph-mon[63909]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:48:16.416 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:48:16.417 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-16T09:48:16.417 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:48:16.417 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:48:16.417 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-16T09:48:16.417 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:48:16.417 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:48:16.417 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-16T09:48:16.417 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 8 2024-09-16T09:48:16.417 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:48:16.417 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-16T09:48:16.417 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:48:16.417 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:48:16.417 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-16T09:48:16.418 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 14 2024-09-16T09:48:16.418 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-16T09:48:16.418 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:48:16.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:16 smithi088 ceph-mon[52758]: from='client.34472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:48:16.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:16 smithi088 ceph-mon[52758]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:48:16.776 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:48:16.777 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f", 2024-09-16T09:48:16.777 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-16T09:48:16.777 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T09:48:16.777 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-16T09:48:16.777 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-16T09:48:16.777 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-09-16T09:48:16.777 INFO:teuthology.orchestra.run.smithi078.stdout: "mds", 2024-09-16T09:48:16.777 INFO:teuthology.orchestra.run.smithi078.stdout: "osd", 2024-09-16T09:48:16.777 INFO:teuthology.orchestra.run.smithi078.stdout: "mon" 2024-09-16T09:48:16.777 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-16T09:48:16.777 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "16/23 daemons upgraded", 2024-09-16T09:48:16.777 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading mds daemons", 2024-09-16T09:48:16.778 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-16T09:48:16.778 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:48:17.222 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-16T09:48:17.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:17 smithi088 ceph-mon[52758]: from='client.34480 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:48:17.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:17 smithi088 ceph-mon[52758]: pgmap v319: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 8.0 KiB/s wr, 14 op/s 2024-09-16T09:48:17.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:17 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/720767114' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:17.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:17 smithi088 ceph-mon[52758]: from='client.34488 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:48:17.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:17 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/2974811808' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:48:17.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:17 smithi078 ceph-mon[63909]: from='client.34480 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:48:17.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:17 smithi078 ceph-mon[63909]: pgmap v319: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 8.0 KiB/s wr, 14 op/s 2024-09-16T09:48:17.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:17 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/720767114' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:17.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:17 smithi078 ceph-mon[63909]: from='client.34488 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:48:17.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:17 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/2974811808' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:48:19.235 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: pgmap v320: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 7.2 KiB/s wr, 11 op/s 2024-09-16T09:48:19.235 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: Detected new or changed devices on smithi088 2024-09-16T09:48:19.235 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:19.236 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:19.236 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:48:19.236 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:48:19.236 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:19.236 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:19.236 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:48:19.236 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:19.236 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:19.236 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:19.236 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:19.236 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:19.236 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: Upgrade: Setting container_image for all mds 2024-09-16T09:48:19.237 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:19.237 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi078.hiotnn"} : dispatch 2024-09-16T09:48:19.237 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi078.hiotnn"}]': finished 2024-09-16T09:48:19.237 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi088.dbunob"} : dispatch 2024-09-16T09:48:19.237 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi088.dbunob"}]': finished 2024-09-16T09:48:19.237 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:19.237 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: Upgrade: Setting container_image for all rgw 2024-09-16T09:48:19.237 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:19.237 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:19.237 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: Upgrade: Setting container_image for all rbd-mirror 2024-09-16T09:48:19.237 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:19.238 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:19.238 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:19.238 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: Upgrade: Setting container_image for all ceph-exporter 2024-09-16T09:48:19.238 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:19.238 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:19.238 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: Upgrade: Setting container_image for all iscsi 2024-09-16T09:48:19.238 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:19 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:19.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: pgmap v320: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 7.2 KiB/s wr, 11 op/s 2024-09-16T09:48:19.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: Detected new or changed devices on smithi088 2024-09-16T09:48:19.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:19.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:19.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:48:19.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:48:19.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:19.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:19.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:48:19.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:19.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:19.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:19.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:19.553 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:19.553 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: Upgrade: Setting container_image for all mds 2024-09-16T09:48:19.553 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:19.553 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi078.hiotnn"} : dispatch 2024-09-16T09:48:19.553 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi078.hiotnn"}]': finished 2024-09-16T09:48:19.553 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi088.dbunob"} : dispatch 2024-09-16T09:48:19.553 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi088.dbunob"}]': finished 2024-09-16T09:48:19.553 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:19.553 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: Upgrade: Setting container_image for all rgw 2024-09-16T09:48:19.553 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:19.553 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:19.553 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: Upgrade: Setting container_image for all rbd-mirror 2024-09-16T09:48:19.554 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:19.554 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:19.554 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:19.554 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: Upgrade: Setting container_image for all ceph-exporter 2024-09-16T09:48:19.554 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:19.554 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:19.554 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: Upgrade: Setting container_image for all iscsi 2024-09-16T09:48:19.554 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:19 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:21.787 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:21 smithi078 ceph-mon[63909]: pgmap v321: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 7.2 KiB/s wr, 9 op/s 2024-09-16T09:48:21.787 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:21 smithi078 ceph-mon[63909]: Upgrade: Updating nfs.foo.0.0.smithi078.wawosh (1/2) 2024-09-16T09:48:21.788 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:21 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:21.788 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:21 smithi078 ceph-mon[63909]: Creating key for client.nfs.foo.0.0.smithi078.wawosh 2024-09-16T09:48:21.788 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:21 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.wawosh", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-16T09:48:21.788 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:21 smithi078 ceph-mon[63909]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-16T09:48:21.788 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:21 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-16T09:48:21.788 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:21 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-16T09:48:21.788 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:21 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:48:21.788 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:21 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-16T09:48:21.788 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:21 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-16T09:48:21.788 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:21 smithi078 ceph-mon[63909]: Rados config object exists: conf-nfs.foo 2024-09-16T09:48:21.788 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:21 smithi078 ceph-mon[63909]: Creating key for client.nfs.foo.0.0.smithi078.wawosh-rgw 2024-09-16T09:48:21.788 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:21 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.wawosh-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-16T09:48:21.789 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:21 smithi078 ceph-mon[63909]: Bind address in nfs.foo.0.0.smithi078.wawosh's ganesha conf is defaulting to empty 2024-09-16T09:48:21.789 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:21 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:48:21.789 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:21 smithi078 ceph-mon[63909]: Deploying daemon nfs.foo.0.0.smithi078.wawosh on smithi078 2024-09-16T09:48:21.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:21 smithi088 ceph-mon[52758]: pgmap v321: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.1 MiB/s rd, 7.2 KiB/s wr, 9 op/s 2024-09-16T09:48:21.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:21 smithi088 ceph-mon[52758]: Upgrade: Updating nfs.foo.0.0.smithi078.wawosh (1/2) 2024-09-16T09:48:21.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:21 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:21.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:21 smithi088 ceph-mon[52758]: Creating key for client.nfs.foo.0.0.smithi078.wawosh 2024-09-16T09:48:21.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:21 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.wawosh", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-16T09:48:21.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:21 smithi088 ceph-mon[52758]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-16T09:48:21.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:21 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-16T09:48:21.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:21 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-16T09:48:21.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:21 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:48:21.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:21 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-16T09:48:21.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:21 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-16T09:48:21.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:21 smithi088 ceph-mon[52758]: Rados config object exists: conf-nfs.foo 2024-09-16T09:48:21.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:21 smithi088 ceph-mon[52758]: Creating key for client.nfs.foo.0.0.smithi078.wawosh-rgw 2024-09-16T09:48:21.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:21 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.wawosh-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-16T09:48:21.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:21 smithi088 ceph-mon[52758]: Bind address in nfs.foo.0.0.smithi078.wawosh's ganesha conf is defaulting to empty 2024-09-16T09:48:21.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:21 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:48:21.802 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:21 smithi088 ceph-mon[52758]: Deploying daemon nfs.foo.0.0.smithi078.wawosh on smithi078 2024-09-16T09:48:23.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:23 smithi088 ceph-mon[52758]: pgmap v322: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 7.1 KiB/s wr, 5 op/s 2024-09-16T09:48:23.573 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:23 smithi078 ceph-mon[63909]: pgmap v322: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 7.1 KiB/s wr, 5 op/s 2024-09-16T09:48:25.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:25 smithi088 ceph-mon[52758]: pgmap v323: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s rd, 7.5 KiB/s wr, 4 op/s 2024-09-16T09:48:25.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:25 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:25.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:25 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:25.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:25 smithi088 ceph-mon[52758]: Evicting (and blocklisting) client session 14598 (172.21.15.78:0/1110678352) 2024-09-16T09:48:25.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:25 smithi088 ceph-mon[52758]: from='mds.? [v2:172.21.15.78:6834/487538732,v1:172.21.15.78:6835/487538732]' entity='mds.foofs.smithi078.hiotnn' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.78:0/1110678352"} : dispatch 2024-09-16T09:48:25.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:25 smithi078 ceph-mon[63909]: pgmap v323: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s rd, 7.5 KiB/s wr, 4 op/s 2024-09-16T09:48:25.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:25 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:25.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:25 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:25.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:25 smithi078 ceph-mon[63909]: Evicting (and blocklisting) client session 14598 (172.21.15.78:0/1110678352) 2024-09-16T09:48:25.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:25 smithi078 ceph-mon[63909]: from='mds.? [v2:172.21.15.78:6834/487538732,v1:172.21.15.78:6835/487538732]' entity='mds.foofs.smithi078.hiotnn' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.78:0/1110678352"} : dispatch 2024-09-16T09:48:26.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:26 smithi088 ceph-mon[52758]: from='mds.? [v2:172.21.15.78:6834/487538732,v1:172.21.15.78:6835/487538732]' entity='mds.foofs.smithi078.hiotnn' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.78:0/1110678352"}]': finished 2024-09-16T09:48:26.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:26 smithi088 ceph-mon[52758]: osdmap e162: 8 total, 8 up, 8 in 2024-09-16T09:48:26.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:26 smithi088 ceph-mon[52758]: Upgrade: Updating nfs.foo.1.0.smithi088.bnccmw (2/2) 2024-09-16T09:48:26.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:26 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:26.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:26 smithi088 ceph-mon[52758]: Creating key for client.nfs.foo.1.0.smithi088.bnccmw 2024-09-16T09:48:26.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:26 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi088.bnccmw", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-16T09:48:26.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:26 smithi088 ceph-mon[52758]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-16T09:48:26.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:26 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-16T09:48:26.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:26 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-16T09:48:26.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:26 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:48:26.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:26 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-16T09:48:26.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:26 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-16T09:48:26.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:26 smithi088 ceph-mon[52758]: Rados config object exists: conf-nfs.foo 2024-09-16T09:48:26.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:26 smithi088 ceph-mon[52758]: Creating key for client.nfs.foo.1.0.smithi088.bnccmw-rgw 2024-09-16T09:48:26.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:26 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi088.bnccmw-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-16T09:48:26.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:26 smithi088 ceph-mon[52758]: Bind address in nfs.foo.1.0.smithi088.bnccmw's ganesha conf is defaulting to empty 2024-09-16T09:48:26.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:26 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:48:26.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:26 smithi088 ceph-mon[52758]: Deploying daemon nfs.foo.1.0.smithi088.bnccmw on smithi088 2024-09-16T09:48:26.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:26 smithi078 ceph-mon[63909]: from='mds.? [v2:172.21.15.78:6834/487538732,v1:172.21.15.78:6835/487538732]' entity='mds.foofs.smithi078.hiotnn' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.78:0/1110678352"}]': finished 2024-09-16T09:48:26.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:26 smithi078 ceph-mon[63909]: osdmap e162: 8 total, 8 up, 8 in 2024-09-16T09:48:26.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:26 smithi078 ceph-mon[63909]: Upgrade: Updating nfs.foo.1.0.smithi088.bnccmw (2/2) 2024-09-16T09:48:26.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:26 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:26.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:26 smithi078 ceph-mon[63909]: Creating key for client.nfs.foo.1.0.smithi088.bnccmw 2024-09-16T09:48:26.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:26 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi088.bnccmw", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-16T09:48:26.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:26 smithi078 ceph-mon[63909]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-16T09:48:26.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:26 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-16T09:48:26.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:26 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-16T09:48:26.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:26 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:48:26.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:26 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-16T09:48:26.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:26 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-16T09:48:26.563 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:26 smithi078 ceph-mon[63909]: Rados config object exists: conf-nfs.foo 2024-09-16T09:48:26.563 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:26 smithi078 ceph-mon[63909]: Creating key for client.nfs.foo.1.0.smithi088.bnccmw-rgw 2024-09-16T09:48:26.563 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:26 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi088.bnccmw-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-16T09:48:26.563 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:26 smithi078 ceph-mon[63909]: Bind address in nfs.foo.1.0.smithi088.bnccmw's ganesha conf is defaulting to empty 2024-09-16T09:48:26.563 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:26 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:48:26.563 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:26 smithi078 ceph-mon[63909]: Deploying daemon nfs.foo.1.0.smithi088.bnccmw on smithi088 2024-09-16T09:48:27.788 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:27 smithi088 ceph-mon[52758]: pgmap v325: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s rd, 614 B/s wr, 1 op/s 2024-09-16T09:48:27.788 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:27 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:48:27.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:27 smithi078 ceph-mon[63909]: pgmap v325: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s rd, 614 B/s wr, 1 op/s 2024-09-16T09:48:27.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:27 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:48:29.950 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:29 smithi078 ceph-mon[63909]: pgmap v326: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s rd, 614 B/s wr, 1 op/s 2024-09-16T09:48:29.950 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:29 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:29.950 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:29 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:29.950 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:29 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:48:29.951 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:29 smithi078 ceph-mon[63909]: from='mds.? [v2:172.21.15.78:6834/487538732,v1:172.21.15.78:6835/487538732]' entity='mds.foofs.smithi078.hiotnn' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.88:0/3730125002"} : dispatch 2024-09-16T09:48:29.963 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:29 smithi088 ceph-mon[52758]: pgmap v326: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s rd, 614 B/s wr, 1 op/s 2024-09-16T09:48:29.963 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:29 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:29.963 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:29 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:29.963 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:29 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:48:29.963 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:29 smithi088 ceph-mon[52758]: from='mds.? [v2:172.21.15.78:6834/487538732,v1:172.21.15.78:6835/487538732]' entity='mds.foofs.smithi078.hiotnn' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.88:0/3730125002"} : dispatch 2024-09-16T09:48:31.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:30 smithi088 ceph-mon[52758]: Evicting (and blocklisting) client session 24337 (172.21.15.88:0/3730125002) 2024-09-16T09:48:31.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:30 smithi088 ceph-mon[52758]: from='mds.? [v2:172.21.15.78:6834/487538732,v1:172.21.15.78:6835/487538732]' entity='mds.foofs.smithi078.hiotnn' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.88:0/3730125002"}]': finished 2024-09-16T09:48:31.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:30 smithi088 ceph-mon[52758]: osdmap e163: 8 total, 8 up, 8 in 2024-09-16T09:48:31.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:30 smithi078 ceph-mon[63909]: Evicting (and blocklisting) client session 24337 (172.21.15.88:0/3730125002) 2024-09-16T09:48:31.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:30 smithi078 ceph-mon[63909]: from='mds.? [v2:172.21.15.78:6834/487538732,v1:172.21.15.78:6835/487538732]' entity='mds.foofs.smithi078.hiotnn' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.88:0/3730125002"}]': finished 2024-09-16T09:48:31.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:30 smithi078 ceph-mon[63909]: osdmap e163: 8 total, 8 up, 8 in 2024-09-16T09:48:31.801 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:31 smithi088 ceph-mon[52758]: pgmap v327: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.9 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-16T09:48:31.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:31 smithi078 ceph-mon[63909]: pgmap v327: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.9 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-16T09:48:33.107 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:33.107 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:33.107 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:32 smithi088 ceph-mon[52758]: pgmap v329: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.2 KiB/s rd, 1.1 KiB/s wr, 3 op/s 2024-09-16T09:48:33.107 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:33.107 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:32 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:33.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:33.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:33.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:32 smithi078 ceph-mon[63909]: pgmap v329: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.2 KiB/s rd, 1.1 KiB/s wr, 3 op/s 2024-09-16T09:48:33.312 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:33.312 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:32 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:35.221 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:34 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:35.221 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:34 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:35.221 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:34 smithi078 ceph-mon[63909]: pgmap v330: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.8 KiB/s wr, 6 op/s 2024-09-16T09:48:35.222 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:34 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:35.222 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:34 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:35.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:34 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:35.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:34 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:35.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:34 smithi088 ceph-mon[52758]: pgmap v330: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.8 KiB/s wr, 6 op/s 2024-09-16T09:48:35.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:34 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:35.302 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:34 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:37.042 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:36 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:37.043 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:36 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:37.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:36 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:37.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:36 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:37.983 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:37 smithi078 ceph-mon[63909]: pgmap v331: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.6 KiB/s wr, 5 op/s 2024-09-16T09:48:38.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:37 smithi088 ceph-mon[52758]: pgmap v331: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.6 KiB/s wr, 5 op/s 2024-09-16T09:48:39.270 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:39 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:39.270 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:39 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:39.270 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:39 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:48:39.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:39 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:48:39.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:39 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:39.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:39 smithi078 ceph-mon[63909]: pgmap v332: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.6 KiB/s wr, 5 op/s 2024-09-16T09:48:39.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:39 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:39.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:39 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:48:39.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:39 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:39.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:39 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:39.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:39 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:39.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:39 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:39.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:39 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:39.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:39 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:39.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:39 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:39.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:39 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:39.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:39 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:39.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:39 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:39.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:39 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:39.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:39 smithi078 ceph-mon[63909]: Upgrade: Setting container_image for all nfs 2024-09-16T09:48:39.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:39 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:39.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:39 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi078.wawosh"} : dispatch 2024-09-16T09:48:39.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:39 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi078.wawosh"}]': finished 2024-09-16T09:48:39.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:39 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi088.bnccmw"} : dispatch 2024-09-16T09:48:39.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:39 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi088.bnccmw"}]': finished 2024-09-16T09:48:39.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:39 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:39.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:39 smithi078 ceph-mon[63909]: Upgrade: Setting container_image for all nvmeof 2024-09-16T09:48:39.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:39 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:39.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:39 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:39.273 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:39 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:39.273 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:39 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:39.279 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:39 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:39.279 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:39 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:39.280 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:39 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:48:39.280 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:39 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:48:39.280 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:39 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:39.280 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:39 smithi088 ceph-mon[52758]: pgmap v332: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.6 KiB/s wr, 5 op/s 2024-09-16T09:48:39.280 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:39 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:39.280 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:39 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:48:39.280 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:39 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:39.280 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:39 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:39.280 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:39 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:39.280 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:39 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:39.280 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:39 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:39.280 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:39 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:39.280 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:39 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:39.281 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:39 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:39.281 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:39 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:39.281 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:39 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:39.281 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:39 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:39.281 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:39 smithi088 ceph-mon[52758]: Upgrade: Setting container_image for all nfs 2024-09-16T09:48:39.281 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:39 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:39.281 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:39 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi078.wawosh"} : dispatch 2024-09-16T09:48:39.281 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:39 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi078.wawosh"}]': finished 2024-09-16T09:48:39.281 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:39 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi088.bnccmw"} : dispatch 2024-09-16T09:48:39.281 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:39 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi088.bnccmw"}]': finished 2024-09-16T09:48:39.281 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:39 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:39.281 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:39 smithi088 ceph-mon[52758]: Upgrade: Setting container_image for all nvmeof 2024-09-16T09:48:39.282 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:39 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:39.282 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:39 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:39.282 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:39 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:39.282 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:39 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:40.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:40 smithi088 ceph-mon[52758]: Upgrade: Updating grafana.smithi078 2024-09-16T09:48:40.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:40 smithi078 ceph-mon[63909]: Upgrade: Updating grafana.smithi078 2024-09-16T09:48:41.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:41 smithi088 ceph-mon[52758]: Deploying daemon grafana.smithi078 on smithi078 2024-09-16T09:48:41.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:41 smithi088 ceph-mon[52758]: pgmap v333: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s rd, 819 B/s wr, 2 op/s 2024-09-16T09:48:41.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:41 smithi078 ceph-mon[63909]: Deploying daemon grafana.smithi078 on smithi078 2024-09-16T09:48:41.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:41 smithi078 ceph-mon[63909]: pgmap v333: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s rd, 819 B/s wr, 2 op/s 2024-09-16T09:48:43.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:43 smithi088 ceph-mon[52758]: pgmap v334: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s rd, 695 B/s wr, 2 op/s 2024-09-16T09:48:43.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:43 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:48:43.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:43 smithi078 ceph-mon[63909]: pgmap v334: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s rd, 695 B/s wr, 2 op/s 2024-09-16T09:48:43.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:43 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:48:45.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:45 smithi088 ceph-mon[52758]: pgmap v335: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.7 KiB/s rd, 1023 B/s wr, 3 op/s 2024-09-16T09:48:45.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:45 smithi078 ceph-mon[63909]: pgmap v335: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.7 KiB/s rd, 1023 B/s wr, 3 op/s 2024-09-16T09:48:47.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:47 smithi088 ceph-mon[52758]: pgmap v336: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 682 B/s rd, 426 B/s wr, 1 op/s 2024-09-16T09:48:47.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:47 smithi078 ceph-mon[63909]: pgmap v336: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 682 B/s rd, 426 B/s wr, 1 op/s 2024-09-16T09:48:47.602 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-16T09:48:48.306 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T09:48:48.306 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (10m) 14s ago 21m 18.8M - 0.27.0 11f11916f8cd 3ebb6832fbcc 2024-09-16T09:48:48.306 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (6m) 14s ago 21m 6899k - 19.3.0-4882-g66e33d6a 261f2dc7b20f 72965213139c 2024-09-16T09:48:48.306 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi088 smithi088 running (6m) 16s ago 19m 6891k - 19.3.0-4882-g66e33d6a 261f2dc7b20f eac0e6855b1b 2024-09-16T09:48:48.307 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (10m) 14s ago 19m 54.0M - dad864ee21e9 4a91b97e29b1 2024-09-16T09:48:48.307 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.hiotnn smithi078 running (57s) 14s ago 13m 28.6M - 19.3.0-4882-g66e33d6a 261f2dc7b20f e3c24b03534b 2024-09-16T09:48:48.307 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi088.dbunob smithi088 running (37s) 16s ago 13m 14.7M - 19.3.0-4882-g66e33d6a 261f2dc7b20f 8ffda444de95 2024-09-16T09:48:48.307 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rrfxge smithi078 *:8443,9283,8765 running (9m) 14s ago 22m 525M - 19.3.0-4882-g66e33d6a 261f2dc7b20f 0d25f2f30f1e 2024-09-16T09:48:48.307 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi088.ncxfdz smithi088 *:8443,9283,8765 running (8m) 16s ago 19m 458M - 19.3.0-4882-g66e33d6a 261f2dc7b20f e2d03f701acd 2024-09-16T09:48:48.307 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (8m) 14s ago 22m 67.6M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f f5a3a0fec757 2024-09-16T09:48:48.307 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi088 smithi088 running (7m) 16s ago 18m 51.8M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f 2e552147327a 2024-09-16T09:48:48.307 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.wawosh smithi078 *:2049 running (24s) 14s ago 13m 64.4M - 5.9 261f2dc7b20f a2e4a24fd2af 2024-09-16T09:48:48.307 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi088.bnccmw smithi088 *:2049 running (19s) 16s ago 13m 23.9M - 5.9 261f2dc7b20f d024a9ce562c 2024-09-16T09:48:48.307 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (9m) 14s ago 19m 11.8M - 1.7.0 72c9c2088986 895ac3ccb2f3 2024-09-16T09:48:48.307 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi088 smithi088 *:9100 running (9m) 16s ago 18m 12.4M - 1.7.0 72c9c2088986 c84a2e50de53 2024-09-16T09:48:48.307 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (6m) 14s ago 18m 397M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 85008b499cb7 2024-09-16T09:48:48.307 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (5m) 14s ago 17m 400M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 12c137fcba70 2024-09-16T09:48:48.308 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (4m) 14s ago 17m 386M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 23a8b69c5c64 2024-09-16T09:48:48.308 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (3m) 14s ago 16m 378M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 00c9aa84d41d 2024-09-16T09:48:48.308 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi088 running (3m) 16s ago 16m 344M 2704M 19.3.0-4882-g66e33d6a 261f2dc7b20f 1d92e66b75d9 2024-09-16T09:48:48.308 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi088 running (2m) 16s ago 16m 328M 2704M 19.3.0-4882-g66e33d6a 261f2dc7b20f 43abc1197fe2 2024-09-16T09:48:48.308 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi088 running (2m) 16s ago 15m 277M 2704M 19.3.0-4882-g66e33d6a 261f2dc7b20f 31302f8f7d8f 2024-09-16T09:48:48.308 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi088 running (85s) 16s ago 15m 288M 2704M 19.3.0-4882-g66e33d6a 261f2dc7b20f f3768fef8040 2024-09-16T09:48:48.308 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (8m) 14s ago 18m 55.7M - 2.51.0 1d3b7f56885b 4c956ead9125 2024-09-16T09:48:48.382 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:48 smithi078 ceph-mon[63909]: from='client.34534 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:48:48.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:48 smithi088 ceph-mon[52758]: from='client.34534 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:48:48.742 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:48:48.743 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-16T09:48:48.743 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:48:48.743 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:48:48.743 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-16T09:48:48.743 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:48:48.743 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:48:48.743 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-16T09:48:48.743 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 8 2024-09-16T09:48:48.743 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:48:48.743 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-16T09:48:48.743 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:48:48.744 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:48:48.744 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-16T09:48:48.744 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 14 2024-09-16T09:48:48.744 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-16T09:48:48.744 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:48:49.099 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:48:49.099 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f", 2024-09-16T09:48:49.099 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-16T09:48:49.099 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-16T09:48:49.099 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-16T09:48:49.099 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-16T09:48:49.099 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-09-16T09:48:49.100 INFO:teuthology.orchestra.run.smithi078.stdout: "mds", 2024-09-16T09:48:49.100 INFO:teuthology.orchestra.run.smithi078.stdout: "nfs", 2024-09-16T09:48:49.100 INFO:teuthology.orchestra.run.smithi078.stdout: "osd", 2024-09-16T09:48:49.100 INFO:teuthology.orchestra.run.smithi078.stdout: "mon" 2024-09-16T09:48:49.100 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-16T09:48:49.100 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "18/23 daemons upgraded", 2024-09-16T09:48:49.100 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading grafana daemons", 2024-09-16T09:48:49.100 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-16T09:48:49.100 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:48:49.385 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:49 smithi078 ceph-mon[63909]: from='client.34538 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:48:49.385 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:49 smithi078 ceph-mon[63909]: pgmap v337: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 682 B/s rd, 341 B/s wr, 1 op/s 2024-09-16T09:48:49.385 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:49 smithi078 ceph-mon[63909]: from='client.34542 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:48:49.385 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:49 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/1669020268' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:49.520 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-16T09:48:49.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:49 smithi088 ceph-mon[52758]: from='client.34538 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:48:49.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:49 smithi088 ceph-mon[52758]: pgmap v337: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 682 B/s rd, 341 B/s wr, 1 op/s 2024-09-16T09:48:49.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:49 smithi088 ceph-mon[52758]: from='client.34542 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:48:49.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:49 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/1669020268' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:48:50.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:50 smithi078 ceph-mon[63909]: from='client.34550 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:48:50.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:50 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/4256444682' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:48:50.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:50 smithi088 ceph-mon[52758]: from='client.34550 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:48:50.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:50 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/4256444682' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:48:51.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:51 smithi078 ceph-mon[63909]: pgmap v338: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 682 B/s rd, 341 B/s wr, 1 op/s 2024-09-16T09:48:51.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:51 smithi088 ceph-mon[52758]: pgmap v338: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 682 B/s rd, 341 B/s wr, 1 op/s 2024-09-16T09:48:53.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:53 smithi078 ceph-mon[63909]: pgmap v339: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 682 B/s rd, 341 B/s wr, 1 op/s 2024-09-16T09:48:53.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:53 smithi088 ceph-mon[52758]: pgmap v339: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 682 B/s rd, 341 B/s wr, 1 op/s 2024-09-16T09:48:54.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:54 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:54.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:54 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:54.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:54 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:48:55.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:54 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:55.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:54 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:48:55.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:54 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:48:55.914 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:55 smithi078 ceph-mon[63909]: pgmap v340: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 853 B/s rd, 341 B/s wr, 1 op/s 2024-09-16T09:48:56.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:55 smithi088 ceph-mon[52758]: pgmap v340: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 853 B/s rd, 341 B/s wr, 1 op/s 2024-09-16T09:48:58.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:57 smithi088 ceph-mon[52758]: pgmap v341: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:48:58.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:57 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:48:58.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:57 smithi078 ceph-mon[63909]: pgmap v341: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:48:58.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:57 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:49:00.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:48:59 smithi088 ceph-mon[52758]: pgmap v342: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:00.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:48:59 smithi078 ceph-mon[63909]: pgmap v342: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:02.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:01 smithi088 ceph-mon[52758]: pgmap v343: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:02.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:01 smithi078 ceph-mon[63909]: pgmap v343: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:03.047 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:02 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:49:03.047 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:02 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:49:03.047 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:02 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:49:03.048 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:02 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:49:03.050 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:02 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:49:03.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:02 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:49:03.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:02 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:49:03.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:02 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:49:04.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:03 smithi088 ceph-mon[52758]: pgmap v344: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:04.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:03 smithi078 ceph-mon[63909]: pgmap v344: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:06.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:05 smithi078 ceph-mon[63909]: pgmap v345: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-16T09:49:06.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:05 smithi088 ceph-mon[52758]: pgmap v345: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-16T09:49:07.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:49:07.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:49:07.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:49:07.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:49:07.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:49:07.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:49:07.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-16T09:49:07.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-16T09:49:07.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:49:07.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.052 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.053 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:49:07.054 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-16T09:49:07.054 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-16T09:49:07.054 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T09:49:07.054 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-16T09:49:07.054 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-16T09:49:07.054 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-16T09:49:07.054 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-16T09:49:07.054 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-16T09:49:07.054 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-16T09:49:07.054 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-16T09:49:07.054 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-16T09:49:07.054 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-16T09:49:07.054 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-16T09:49:07.055 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-16T09:49:07.055 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T09:49:07.055 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-16T09:49:07.055 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-16T09:49:07.055 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-16T09:49:07.055 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-16T09:49:07.055 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-16T09:49:07.055 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-16T09:49:07.055 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-16T09:49:07.055 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-16T09:49:07.055 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T09:49:07.055 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T09:49:07.055 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T09:49:07.056 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T09:49:07.056 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T09:49:07.056 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T09:49:07.056 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-16T09:49:07.056 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-16T09:49:07.056 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:49:07.056 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:49:07.056 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:49:07.056 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:49:07.056 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:49:07.056 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:49:07.056 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:49:07.057 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:49:07.057 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:49:07.057 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:06 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:49:07.312 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:49:07.312 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:49:07.312 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:49:07.312 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:49:07.312 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:49:07.312 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:49:07.312 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-16T09:49:07.313 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-16T09:49:07.313 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:49:07.313 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.313 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.313 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.313 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.313 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.313 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.313 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.313 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.313 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.313 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.314 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.314 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.314 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.314 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.314 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.314 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.314 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.314 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:07.314 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:49:07.314 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-16T09:49:07.314 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-16T09:49:07.314 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T09:49:07.314 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-16T09:49:07.314 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-16T09:49:07.315 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-16T09:49:07.315 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-16T09:49:07.315 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-16T09:49:07.315 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-16T09:49:07.315 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-16T09:49:07.315 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-16T09:49:07.315 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-16T09:49:07.315 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-16T09:49:07.315 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-16T09:49:07.316 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T09:49:07.316 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-16T09:49:07.316 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-16T09:49:07.316 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-16T09:49:07.316 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-16T09:49:07.316 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-16T09:49:07.316 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-16T09:49:07.316 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-16T09:49:07.316 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-16T09:49:07.316 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T09:49:07.316 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T09:49:07.316 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T09:49:07.317 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T09:49:07.317 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T09:49:07.317 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-16T09:49:07.317 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-16T09:49:07.317 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-16T09:49:07.317 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:49:07.317 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:49:07.317 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:49:07.317 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:49:07.317 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:49:07.317 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:49:07.317 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:49:07.317 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:49:07.318 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:49:07.318 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:06 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:49:08.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:07 smithi088 ceph-mon[52758]: pgmap v346: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:08.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:07 smithi088 ceph-mon[52758]: Upgrade: Finalizing container_image settings 2024-09-16T09:49:08.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:07 smithi088 ceph-mon[52758]: Upgrade: Complete! 2024-09-16T09:49:08.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:07 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:49:08.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:07 smithi078 ceph-mon[63909]: pgmap v346: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:08.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:07 smithi078 ceph-mon[63909]: Upgrade: Finalizing container_image settings 2024-09-16T09:49:08.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:07 smithi078 ceph-mon[63909]: Upgrade: Complete! 2024-09-16T09:49:08.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:07 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:49:10.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:09 smithi088 ceph-mon[52758]: pgmap v347: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:10.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:09 smithi078 ceph-mon[63909]: pgmap v347: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:12.296 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:11 smithi078 ceph-mon[63909]: pgmap v348: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:12.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:11 smithi088 ceph-mon[52758]: pgmap v348: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:13.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:12 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:49:13.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:12 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:49:14.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:13 smithi088 ceph-mon[52758]: pgmap v349: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:14.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:13 smithi078 ceph-mon[63909]: pgmap v349: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:16.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:15 smithi088 ceph-mon[52758]: pgmap v350: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-16T09:49:16.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:15 smithi078 ceph-mon[63909]: pgmap v350: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-16T09:49:17.301 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:16 smithi088 ceph-mon[52758]: pgmap v351: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:17.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:16 smithi078 ceph-mon[63909]: pgmap v351: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:19.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:19 smithi088 ceph-mon[52758]: pgmap v352: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:19.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:19 smithi078 ceph-mon[63909]: pgmap v352: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:20.289 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:20 smithi078 ceph-mon[63909]: from='client.34558 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:49:20.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:20 smithi088 ceph-mon[52758]: from='client.34558 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:49:20.702 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -e sha1=66e33d6af1496291b0602ee0a859e68145f17dad -- bash -c 'ceph orch ps' 2024-09-16T09:49:21.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:21 smithi078 ceph-mon[63909]: pgmap v353: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:21.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:21 smithi088 ceph-mon[52758]: pgmap v353: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:22.753 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T09:49:22.753 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (10m) 20s ago 21m 18.8M - 0.27.0 11f11916f8cd 3ebb6832fbcc 2024-09-16T09:49:22.753 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (7m) 20s ago 21m 6899k - 19.3.0-4882-g66e33d6a 261f2dc7b20f 72965213139c 2024-09-16T09:49:22.754 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi088 smithi088 running (7m) 50s ago 19m 6891k - 19.3.0-4882-g66e33d6a 261f2dc7b20f eac0e6855b1b 2024-09-16T09:49:22.754 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (29s) 20s ago 20m 35.3M - 10.4.0 c8b91775d855 5b8d7af09ad5 2024-09-16T09:49:22.754 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.hiotnn smithi078 running (92s) 20s ago 14m 28.8M - 19.3.0-4882-g66e33d6a 261f2dc7b20f e3c24b03534b 2024-09-16T09:49:22.754 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi088.dbunob smithi088 running (71s) 50s ago 14m 14.7M - 19.3.0-4882-g66e33d6a 261f2dc7b20f 8ffda444de95 2024-09-16T09:49:22.754 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rrfxge smithi078 *:8443,9283,8765 running (9m) 20s ago 22m 533M - 19.3.0-4882-g66e33d6a 261f2dc7b20f 0d25f2f30f1e 2024-09-16T09:49:22.754 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi088.ncxfdz smithi088 *:8443,9283,8765 running (9m) 50s ago 19m 458M - 19.3.0-4882-g66e33d6a 261f2dc7b20f e2d03f701acd 2024-09-16T09:49:22.754 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (8m) 20s ago 23m 71.6M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f f5a3a0fec757 2024-09-16T09:49:22.754 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi088 smithi088 running (8m) 50s ago 19m 51.8M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f 2e552147327a 2024-09-16T09:49:22.754 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.wawosh smithi078 *:2049 running (59s) 20s ago 13m 65.3M - 5.9 261f2dc7b20f a2e4a24fd2af 2024-09-16T09:49:22.754 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi088.bnccmw smithi088 *:2049 running (54s) 50s ago 13m 23.9M - 5.9 261f2dc7b20f d024a9ce562c 2024-09-16T09:49:22.754 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (10m) 20s ago 20m 11.5M - 1.7.0 72c9c2088986 895ac3ccb2f3 2024-09-16T09:49:22.754 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi088 smithi088 *:9100 running (10m) 50s ago 19m 12.4M - 1.7.0 72c9c2088986 c84a2e50de53 2024-09-16T09:49:22.755 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (6m) 20s ago 18m 397M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 85008b499cb7 2024-09-16T09:49:22.755 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (6m) 20s ago 18m 400M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 12c137fcba70 2024-09-16T09:49:22.755 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (5m) 20s ago 17m 386M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 23a8b69c5c64 2024-09-16T09:49:22.755 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (4m) 20s ago 17m 378M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 00c9aa84d41d 2024-09-16T09:49:22.755 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi088 running (3m) 50s ago 16m 344M 2704M 19.3.0-4882-g66e33d6a 261f2dc7b20f 1d92e66b75d9 2024-09-16T09:49:22.755 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi088 running (3m) 50s ago 16m 328M 2704M 19.3.0-4882-g66e33d6a 261f2dc7b20f 43abc1197fe2 2024-09-16T09:49:22.755 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi088 running (2m) 50s ago 16m 277M 2704M 19.3.0-4882-g66e33d6a 261f2dc7b20f 31302f8f7d8f 2024-09-16T09:49:22.755 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi088 running (2m) 50s ago 15m 288M 2704M 19.3.0-4882-g66e33d6a 261f2dc7b20f f3768fef8040 2024-09-16T09:49:22.755 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (9m) 20s ago 19m 56.1M - 2.51.0 1d3b7f56885b 4c956ead9125 2024-09-16T09:49:23.429 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:23 smithi078 ceph-mon[63909]: pgmap v354: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:23.429 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:23 smithi078 ceph-mon[63909]: from='client.34562 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:49:23.452 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -e sha1=66e33d6af1496291b0602ee0a859e68145f17dad -- bash -c 'ceph versions' 2024-09-16T09:49:23.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:23 smithi088 ceph-mon[52758]: pgmap v354: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:23.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:23 smithi088 ceph-mon[52758]: from='client.34562 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:49:25.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:25 smithi078 ceph-mon[63909]: pgmap v355: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-16T09:49:25.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:25 smithi088 ceph-mon[52758]: pgmap v355: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-16T09:49:25.608 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:49:25.608 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-16T09:49:25.608 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:49:25.608 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:49:25.608 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-16T09:49:25.608 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:49:25.608 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:49:25.608 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-16T09:49:25.608 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 8 2024-09-16T09:49:25.609 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:49:25.609 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-16T09:49:25.609 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:49:25.609 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:49:25.609 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-16T09:49:25.609 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 14 2024-09-16T09:49:25.609 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-16T09:49:25.609 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:49:26.468 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:26 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/4109046792' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:26.491 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -e sha1=66e33d6af1496291b0602ee0a859e68145f17dad -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-09-16T09:49:26.522 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:26 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/4109046792' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:49:27.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:27 smithi088 ceph-mon[52758]: pgmap v356: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:27.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:27 smithi078 ceph-mon[63909]: pgmap v356: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:28.438 INFO:teuthology.orchestra.run.smithi078.stdout:wait for servicemap items w/ changing names to refresh 2024-09-16T09:49:28.438 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:28 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:49:28.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:28 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:49:29.142 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -e sha1=66e33d6af1496291b0602ee0a859e68145f17dad -- bash -c 'sleep 60' 2024-09-16T09:49:29.439 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:29 smithi078 ceph-mon[63909]: pgmap v357: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:29.550 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:29 smithi088 ceph-mon[52758]: pgmap v357: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:31.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:31 smithi088 ceph-mon[52758]: pgmap v358: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:31.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:31 smithi078 ceph-mon[63909]: pgmap v358: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:33.550 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:33 smithi088 ceph-mon[52758]: pgmap v359: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:33.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:33 smithi078 ceph-mon[63909]: pgmap v359: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:35.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:35 smithi088 ceph-mon[52758]: pgmap v360: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-16T09:49:35.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:35 smithi078 ceph-mon[63909]: pgmap v360: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-16T09:49:37.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:37 smithi088 ceph-mon[52758]: pgmap v361: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:37.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:37 smithi078 ceph-mon[63909]: pgmap v361: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:39.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:39 smithi088 ceph-mon[52758]: pgmap v362: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:39.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:39 smithi078 ceph-mon[63909]: pgmap v362: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:41.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:41 smithi088 ceph-mon[52758]: pgmap v363: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:41.564 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:41 smithi078 ceph-mon[63909]: pgmap v363: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:43.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:43 smithi088 ceph-mon[52758]: pgmap v364: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:43.552 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:43 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:49:43.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:43 smithi078 ceph-mon[63909]: pgmap v364: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:43.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:43 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:49:45.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:45 smithi088 ceph-mon[52758]: pgmap v365: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-16T09:49:45.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:45 smithi078 ceph-mon[63909]: pgmap v365: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-16T09:49:47.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:47 smithi088 ceph-mon[52758]: pgmap v366: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:47.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:47 smithi078 ceph-mon[63909]: pgmap v366: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:49.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:49 smithi088 ceph-mon[52758]: pgmap v367: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:49.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:49 smithi078 ceph-mon[63909]: pgmap v367: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:51.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:51 smithi088 ceph-mon[52758]: pgmap v368: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:51.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:51 smithi078 ceph-mon[63909]: pgmap v368: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:53.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:53 smithi088 ceph-mon[52758]: pgmap v369: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:53.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:53 smithi078 ceph-mon[63909]: pgmap v369: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:55.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:55 smithi088 ceph-mon[52758]: pgmap v370: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-16T09:49:55.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:55 smithi078 ceph-mon[63909]: pgmap v370: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-16T09:49:57.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:57 smithi088 ceph-mon[52758]: pgmap v371: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:57.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:57 smithi078 ceph-mon[63909]: pgmap v371: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:58.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:58 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:49:58.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:58 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:49:59.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:49:59 smithi088 ceph-mon[52758]: pgmap v372: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:49:59.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:49:59 smithi078 ceph-mon[63909]: pgmap v372: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:00.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:00 smithi088 ceph-mon[52758]: overall HEALTH_OK 2024-09-16T09:50:00.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:00 smithi078 ceph-mon[63909]: overall HEALTH_OK 2024-09-16T09:50:01.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:01 smithi088 ceph-mon[52758]: pgmap v373: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:01.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:01 smithi078 ceph-mon[63909]: pgmap v373: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:03.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:03 smithi088 ceph-mon[52758]: pgmap v374: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:03.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:03 smithi078 ceph-mon[63909]: pgmap v374: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:05.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:05 smithi088 ceph-mon[52758]: pgmap v375: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-16T09:50:05.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:05 smithi078 ceph-mon[63909]: pgmap v375: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-16T09:50:07.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:07 smithi088 ceph-mon[52758]: pgmap v376: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:07.551 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:07 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:50:07.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:07 smithi078 ceph-mon[63909]: pgmap v376: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:07.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:07 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-16T09:50:08.553 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:08 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:50:08.553 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:08 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:50:08.553 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:08 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:50:08.553 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:08 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:50:08.553 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:08 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:50:08.553 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:08 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:50:08.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:08 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:50:08.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:08 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:50:08.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:08 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-16T09:50:08.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:08 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-16T09:50:08.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:08 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:50:08.562 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:08 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' 2024-09-16T09:50:10.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:09 smithi088 ceph-mon[52758]: pgmap v377: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:10.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:09 smithi078 ceph-mon[63909]: pgmap v377: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:12.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:11 smithi088 ceph-mon[52758]: pgmap v378: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:12.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:11 smithi078 ceph-mon[63909]: pgmap v378: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:13.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:12 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:50:13.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:12 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:50:14.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:13 smithi088 ceph-mon[52758]: pgmap v379: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:14.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:13 smithi078 ceph-mon[63909]: pgmap v379: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:16.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:15 smithi088 ceph-mon[52758]: pgmap v380: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-16T09:50:16.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:15 smithi078 ceph-mon[63909]: pgmap v380: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-16T09:50:18.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:17 smithi088 ceph-mon[52758]: pgmap v381: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:18.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:17 smithi078 ceph-mon[63909]: pgmap v381: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:20.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:19 smithi088 ceph-mon[52758]: pgmap v382: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:20.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:19 smithi078 ceph-mon[63909]: pgmap v382: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:22.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:21 smithi088 ceph-mon[52758]: pgmap v383: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:22.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:21 smithi078 ceph-mon[63909]: pgmap v383: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:24.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:23 smithi088 ceph-mon[52758]: pgmap v384: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:24.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:23 smithi078 ceph-mon[63909]: pgmap v384: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:26.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:25 smithi088 ceph-mon[52758]: pgmap v385: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-16T09:50:26.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:25 smithi078 ceph-mon[63909]: pgmap v385: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-16T09:50:28.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:27 smithi088 ceph-mon[52758]: pgmap v386: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:28.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:27 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:50:28.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:27 smithi078 ceph-mon[63909]: pgmap v386: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:28.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:27 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:50:30.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:29 smithi088 ceph-mon[52758]: pgmap v387: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:30.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:29 smithi078 ceph-mon[63909]: pgmap v387: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:31.674 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -e sha1=66e33d6af1496291b0602ee0a859e68145f17dad -- bash -c 'ceph orch ps' 2024-09-16T09:50:31.974 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:31 smithi078 ceph-mon[63909]: pgmap v388: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:32.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:31 smithi088 ceph-mon[52758]: pgmap v388: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:33.670 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T09:50:33.671 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (12m) 91s ago 22m 18.8M - 0.27.0 11f11916f8cd 3ebb6832fbcc 2024-09-16T09:50:33.671 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (8m) 91s ago 22m 6899k - 19.3.0-4882-g66e33d6a 261f2dc7b20f 72965213139c 2024-09-16T09:50:33.671 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi088 smithi088 running (8m) 2m ago 20m 6891k - 19.3.0-4882-g66e33d6a 261f2dc7b20f eac0e6855b1b 2024-09-16T09:50:33.671 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (100s) 91s ago 21m 35.3M - 10.4.0 c8b91775d855 5b8d7af09ad5 2024-09-16T09:50:33.671 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.hiotnn smithi078 running (2m) 91s ago 15m 28.8M - 19.3.0-4882-g66e33d6a 261f2dc7b20f e3c24b03534b 2024-09-16T09:50:33.671 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi088.dbunob smithi088 running (2m) 2m ago 15m 14.7M - 19.3.0-4882-g66e33d6a 261f2dc7b20f 8ffda444de95 2024-09-16T09:50:33.671 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.rrfxge smithi078 *:8443,9283,8765 running (11m) 91s ago 24m 533M - 19.3.0-4882-g66e33d6a 261f2dc7b20f 0d25f2f30f1e 2024-09-16T09:50:33.671 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi088.ncxfdz smithi088 *:8443,9283,8765 running (10m) 2m ago 20m 458M - 19.3.0-4882-g66e33d6a 261f2dc7b20f e2d03f701acd 2024-09-16T09:50:33.671 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (10m) 91s ago 24m 71.6M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f f5a3a0fec757 2024-09-16T09:50:33.671 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi088 smithi088 running (9m) 2m ago 20m 51.8M 2048M 19.3.0-4882-g66e33d6a 261f2dc7b20f 2e552147327a 2024-09-16T09:50:33.671 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.wawosh smithi078 *:2049 running (2m) 91s ago 15m 65.3M - 5.9 261f2dc7b20f a2e4a24fd2af 2024-09-16T09:50:33.672 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi088.bnccmw smithi088 *:2049 running (2m) 2m ago 15m 23.9M - 5.9 261f2dc7b20f d024a9ce562c 2024-09-16T09:50:33.672 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (11m) 91s ago 21m 11.5M - 1.7.0 72c9c2088986 895ac3ccb2f3 2024-09-16T09:50:33.672 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi088 smithi088 *:9100 running (11m) 2m ago 20m 12.4M - 1.7.0 72c9c2088986 c84a2e50de53 2024-09-16T09:50:33.672 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (8m) 91s ago 19m 397M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 85008b499cb7 2024-09-16T09:50:33.672 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (7m) 91s ago 19m 400M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 12c137fcba70 2024-09-16T09:50:33.672 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (6m) 91s ago 18m 386M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 23a8b69c5c64 2024-09-16T09:50:33.672 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (5m) 91s ago 18m 378M 1936M 19.3.0-4882-g66e33d6a 261f2dc7b20f 00c9aa84d41d 2024-09-16T09:50:33.672 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi088 running (4m) 2m ago 18m 344M 2704M 19.3.0-4882-g66e33d6a 261f2dc7b20f 1d92e66b75d9 2024-09-16T09:50:33.672 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi088 running (4m) 2m ago 17m 328M 2704M 19.3.0-4882-g66e33d6a 261f2dc7b20f 43abc1197fe2 2024-09-16T09:50:33.672 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi088 running (3m) 2m ago 17m 277M 2704M 19.3.0-4882-g66e33d6a 261f2dc7b20f 31302f8f7d8f 2024-09-16T09:50:33.672 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi088 running (3m) 2m ago 16m 288M 2704M 19.3.0-4882-g66e33d6a 261f2dc7b20f f3768fef8040 2024-09-16T09:50:33.672 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (10m) 91s ago 20m 56.1M - 2.51.0 1d3b7f56885b 4c956ead9125 2024-09-16T09:50:33.984 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:33 smithi078 ceph-mon[63909]: pgmap v389: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:34.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:33 smithi088 ceph-mon[52758]: pgmap v389: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:34.349 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -e sha1=66e33d6af1496291b0602ee0a859e68145f17dad -- bash -c 'ceph orch upgrade status' 2024-09-16T09:50:34.971 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:34 smithi078 ceph-mon[63909]: from='client.34570 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:50:35.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:34 smithi088 ceph-mon[52758]: from='client.34570 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:50:35.890 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:35 smithi078 ceph-mon[63909]: pgmap v390: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-16T09:50:36.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:35 smithi088 ceph-mon[52758]: pgmap v390: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-16T09:50:36.360 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:50:36.360 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": null, 2024-09-16T09:50:36.360 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": false, 2024-09-16T09:50:36.360 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "", 2024-09-16T09:50:36.360 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-09-16T09:50:36.360 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": null, 2024-09-16T09:50:36.360 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "", 2024-09-16T09:50:36.360 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-16T09:50:36.361 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:50:36.964 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -e sha1=66e33d6af1496291b0602ee0a859e68145f17dad -- bash -c 'ceph health detail' 2024-09-16T09:50:37.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:37 smithi078 ceph-mon[63909]: pgmap v391: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:37.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:37 smithi078 ceph-mon[63909]: from='client.34574 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:50:38.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:37 smithi088 ceph-mon[52758]: pgmap v391: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:38.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:37 smithi088 ceph-mon[52758]: from='client.34574 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:50:39.071 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-16T09:50:39.677 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:39 smithi078 ceph-mon[63909]: pgmap v392: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:39.677 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:39 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/1482073242' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:50:39.678 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -e sha1=66e33d6af1496291b0602ee0a859e68145f17dad -- bash -c 'ceph versions' 2024-09-16T09:50:40.054 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:39 smithi088 ceph-mon[52758]: pgmap v392: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:40.054 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:39 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/1482073242' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-16T09:50:41.784 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-16T09:50:41.784 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-16T09:50:41.784 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:50:41.784 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:50:41.784 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-16T09:50:41.784 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:50:41.784 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:50:41.784 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-16T09:50:41.785 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 8 2024-09-16T09:50:41.785 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:50:41.785 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-16T09:50:41.785 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 2 2024-09-16T09:50:41.785 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:50:41.785 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-16T09:50:41.785 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)": 14 2024-09-16T09:50:41.785 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-16T09:50:41.785 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-16T09:50:42.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:41 smithi088 ceph-mon[52758]: pgmap v393: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:42.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:41 smithi078 ceph-mon[63909]: pgmap v393: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:42.470 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -e sha1=66e33d6af1496291b0602ee0a859e68145f17dad -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-09-16T09:50:42.769 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:42 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/632984896' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:50:42.769 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:42 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:50:43.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:42 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/632984896' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:50:43.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:42 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:50:43.779 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:43 smithi078 ceph-mon[63909]: pgmap v394: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:44.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:43 smithi088 ceph-mon[52758]: pgmap v394: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:44.602 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-16T09:50:44.986 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:44 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/740077608' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:50:45.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:44 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/740077608' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:50:45.302 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -e sha1=66e33d6af1496291b0602ee0a859e68145f17dad -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-09-16T09:50:45.873 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:45 smithi078 ceph-mon[63909]: pgmap v395: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-16T09:50:46.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:45 smithi088 ceph-mon[52758]: pgmap v395: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-16T09:50:47.452 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-4882-g66e33d6a (66e33d6af1496291b0602ee0a859e68145f17dad) squid (dev)" 2024-09-16T09:50:47.758 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:47 smithi078 ceph-mon[63909]: pgmap v396: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:47.758 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:47 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/3216372572' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:50:48.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:47 smithi088 ceph-mon[52758]: pgmap v396: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:48.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:47 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/3216372572' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-16T09:50:48.092 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-09-16T09:50:48.093 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-09-16T09:50:48.094 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-16T09:50:50.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:49 smithi088 ceph-mon[52758]: pgmap v397: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:50.058 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-16T09:50:50.059 INFO:teuthology.orchestra.run.smithi078.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-16T09:27:01.706525Z", "last_refresh": "2024-09-16T09:49:01.781055Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-16T09:26:56.078865Z", "last_refresh": "2024-09-16T09:48:31.898223Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-09-16T09:26:58.846101Z", "last_refresh": "2024-09-16T09:49:01.781204Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-16T09:35:01.639512Z", "last_refresh": "2024-09-16T09:48:31.898806Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-16T09:26:54.609387Z", "last_refresh": "2024-09-16T09:48:31.898319Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi078:172.21.15.78=smithi078", "smithi088:172.21.15.88=smithi088"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-16T09:27:44.045718Z", "last_refresh": "2024-09-16T09:48:31.898391Z", "running": 2, "size": 2}}, {"events": ["2024-09-16T09:50:07.595820Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-09-16T09:35:23.508570Z", "last_refresh": "2024-09-16T09:48:31.898885Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-16T09:27:00.481998Z", "last_refresh": "2024-09-16T09:48:31.898462Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f", "last_refresh": "2024-09-16T09:48:31.898530Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-16T09:26:57.619957Z", "last_refresh": "2024-09-16T09:49:01.781332Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-16T09:50:50.059 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:49 smithi078 ceph-mon[63909]: pgmap v397: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:50.675 INFO:tasks.cephadm:nfs.foo has 2/2 2024-09-16T09:50:50.675 DEBUG:teuthology.parallel:result is None 2024-09-16T09:50:50.675 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-16T09:50:50.687 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-16T09:50:50.688 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-09-16T09:50:50.714 INFO:teuthology.orchestra.run.smithi078.stderr:+ umount /mnt/foo 2024-09-16T09:50:50.739 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-16T09:50:50.750 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-16T09:50:50.750 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-09-16T09:50:52.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:51 smithi088 ceph-mon[52758]: from='client.34594 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T09:50:52.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:51 smithi088 ceph-mon[52758]: pgmap v398: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:52.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:51 smithi078 ceph-mon[63909]: from='client.34594 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T09:50:52.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:51 smithi078 ceph-mon[63909]: pgmap v398: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:52.838 INFO:teuthology.orchestra.run.smithi078.stdout: "foo" 2024-09-16T09:50:53.429 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs export ls foo --detailed' 2024-09-16T09:50:53.941 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:53 smithi078 ceph-mon[63909]: pgmap v399: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:53.941 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:53 smithi078 ceph-mon[63909]: from='client.34598 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:50:54.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:53 smithi088 ceph-mon[52758]: pgmap v399: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:54.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:53 smithi088 ceph-mon[52758]: from='client.34598 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:50:55.455 INFO:teuthology.orchestra.run.smithi078.stdout:[ 2024-09-16T09:50:55.455 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-09-16T09:50:55.455 INFO:teuthology.orchestra.run.smithi078.stdout: "access_type": "RW", 2024-09-16T09:50:55.455 INFO:teuthology.orchestra.run.smithi078.stdout: "clients": [], 2024-09-16T09:50:55.455 INFO:teuthology.orchestra.run.smithi078.stdout: "cluster_id": "foo", 2024-09-16T09:50:55.456 INFO:teuthology.orchestra.run.smithi078.stdout: "export_id": 1, 2024-09-16T09:50:55.456 INFO:teuthology.orchestra.run.smithi078.stdout: "fsal": { 2024-09-16T09:50:55.456 INFO:teuthology.orchestra.run.smithi078.stdout: "fs_name": "foofs", 2024-09-16T09:50:55.456 INFO:teuthology.orchestra.run.smithi078.stdout: "name": "CEPH", 2024-09-16T09:50:55.456 INFO:teuthology.orchestra.run.smithi078.stdout: "user_id": "nfs.foo.1" 2024-09-16T09:50:55.456 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-16T09:50:55.456 INFO:teuthology.orchestra.run.smithi078.stdout: "path": "/", 2024-09-16T09:50:55.456 INFO:teuthology.orchestra.run.smithi078.stdout: "protocols": [ 2024-09-16T09:50:55.456 INFO:teuthology.orchestra.run.smithi078.stdout: 4 2024-09-16T09:50:55.456 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-16T09:50:55.456 INFO:teuthology.orchestra.run.smithi078.stdout: "pseudo": "/fake", 2024-09-16T09:50:55.456 INFO:teuthology.orchestra.run.smithi078.stdout: "security_label": true, 2024-09-16T09:50:55.456 INFO:teuthology.orchestra.run.smithi078.stdout: "squash": "none", 2024-09-16T09:50:55.456 INFO:teuthology.orchestra.run.smithi078.stdout: "transports": [ 2024-09-16T09:50:55.457 INFO:teuthology.orchestra.run.smithi078.stdout: "TCP" 2024-09-16T09:50:55.457 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-09-16T09:50:55.457 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-16T09:50:55.457 INFO:teuthology.orchestra.run.smithi078.stdout:] 2024-09-16T09:50:55.778 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:55 smithi078 ceph-mon[63909]: pgmap v400: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-16T09:50:56.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:55 smithi088 ceph-mon[52758]: pgmap v400: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-16T09:50:56.089 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- bash -c 'rados -p .nfs --all ls -' 2024-09-16T09:50:57.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:56 smithi088 ceph-mon[52758]: from='client.34602 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:50:57.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:56 smithi078 ceph-mon[63909]: from='client.34602 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T09:50:57.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:57 smithi078 ceph-mon[63909]: pgmap v401: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:57.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:57 smithi078 ceph-mon[63909]: mgrmap e44: smithi078.rrfxge(active, since 10m), standbys: smithi088.ncxfdz 2024-09-16T09:50:57.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:57 smithi078 ceph-mon[63909]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:50:58.013 INFO:teuthology.orchestra.run.smithi078.stdout:foo rec-0000000000000004:nfs.foo.1 2024-09-16T09:50:58.014 INFO:teuthology.orchestra.run.smithi078.stdout:foo grace 2024-09-16T09:50:58.014 INFO:teuthology.orchestra.run.smithi078.stdout:foo rec-0000000000000004:nfs.foo.0 2024-09-16T09:50:58.014 INFO:teuthology.orchestra.run.smithi078.stdout:foo conf-nfs.foo 2024-09-16T09:50:58.014 INFO:teuthology.orchestra.run.smithi078.stdout:foo export-1 2024-09-16T09:50:58.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:57 smithi088 ceph-mon[52758]: pgmap v401: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:50:58.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:57 smithi088 ceph-mon[52758]: mgrmap e44: smithi078.rrfxge(active, since 10m), standbys: smithi088.ncxfdz 2024-09-16T09:50:58.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:57 smithi088 ceph-mon[52758]: from='mgr.24621 172.21.15.78:0/617081398' entity='mgr.smithi078.rrfxge' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-16T09:50:58.616 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-16T09:50:58.627 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-16T09:50:58.627 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a70c765e-740d-11ef-bceb-c7b262605968 -- bash -c 'set -ex 2024-09-16T09:50:58.627 DEBUG:teuthology.orchestra.run.smithi078:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-09-16T09:50:58.627 DEBUG:teuthology.orchestra.run.smithi078:> ' 2024-09-16T09:50:59.908 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:50:59 smithi078 ceph-mon[63909]: pgmap v402: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:51:00.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:50:59 smithi088 ceph-mon[52758]: pgmap v402: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-16T09:51:00.394 INFO:teuthology.orchestra.run.smithi078.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-09-16T09:51:00.657 INFO:teuthology.orchestra.run.smithi078.stderr:+ [[ 8 -gt 2 ]] 2024-09-16T09:51:00.948 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:51:00 smithi078 ceph-mon[63909]: from='client.? 172.21.15.78:0/3670609168' entity='client.admin' cmd={"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"} : dispatch 2024-09-16T09:51:01.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:51:00 smithi088 ceph-mon[52758]: from='client.? 172.21.15.78:0/3670609168' entity='client.admin' cmd={"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"} : dispatch 2024-09-16T09:51:01.290 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-16T09:51:01.303 INFO:tasks.cephadm:Teardown begin 2024-09-16T09:51:01.304 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-16T09:51:01.332 DEBUG:teuthology.orchestra.run.smithi088:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-16T09:51:01.366 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-16T09:51:01.366 DEBUG:teuthology.orchestra.run.smithi078:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-16T09:51:01.388 DEBUG:teuthology.orchestra.run.smithi088:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-16T09:51:01.425 INFO:tasks.cephadm:Stopping all daemons... 2024-09-16T09:51:01.426 INFO:tasks.cephadm.mon.smithi078:Stopping mon.smithi078... 2024-09-16T09:51:01.426 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-a70c765e-740d-11ef-bceb-c7b262605968@mon.smithi078 2024-09-16T09:51:01.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:51:01 smithi078 systemd[1]: Stopping Ceph mon.smithi078 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:51:02.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:51:01 smithi088 ceph-mon[52758]: pgmap v403: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T09:51:02.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:51:01 smithi078 ceph-mon[63909]: pgmap v403: 97 pgs: 97 active+clean; 240 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-16T09:51:02.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:51:01 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi078[63905]: 2024-09-16T09:51:01.727+0000 7f4794753640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi078 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T09:51:02.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:51:01 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi078[63905]: 2024-09-16T09:51:01.727+0000 7f4794753640 -1 mon.smithi078@0(leader) e3 *** Got Signal Terminated *** 2024-09-16T09:51:02.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:51:01 smithi078 podman[93808]: 2024-09-16 09:51:01.829568576 +0000 UTC m=+0.218100995 container died f5a3a0fec7570aa41f61afb64ca6a348ab03e2bc531672b34a47a0135b944c31 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi078, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-16T09:51:02.362 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:51:02 smithi078 podman[93808]: 2024-09-16 09:51:02.343790095 +0000 UTC m=+0.732322502 container cleanup f5a3a0fec7570aa41f61afb64ca6a348ab03e2bc531672b34a47a0135b944c31 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi078, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-16T09:51:02.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:51:02 smithi078 bash[93808]: ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi078 2024-09-16T09:51:02.772 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a70c765e-740d-11ef-bceb-c7b262605968@mon.smithi078.service' 2024-09-16T09:51:02.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:51:02 smithi078 podman[93834]: 2024-09-16 09:51:02.553507431 +0000 UTC m=+0.720347708 container remove f5a3a0fec7570aa41f61afb64ca6a348ab03e2bc531672b34a47a0135b944c31 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi078, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_CLEAN=True) 2024-09-16T09:51:02.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:51:02 smithi078 systemd[1]: ceph-a70c765e-740d-11ef-bceb-c7b262605968@mon.smithi078.service: Deactivated successfully. 2024-09-16T09:51:02.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:51:02 smithi078 systemd[1]: Stopped Ceph mon.smithi078 for a70c765e-740d-11ef-bceb-c7b262605968. 2024-09-16T09:51:02.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 16 09:51:02 smithi078 systemd[1]: ceph-a70c765e-740d-11ef-bceb-c7b262605968@mon.smithi078.service: Consumed 13.239s CPU time. 2024-09-16T09:51:02.832 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-16T09:51:02.832 INFO:tasks.cephadm.mon.smithi078:Stopped mon.smithi078 2024-09-16T09:51:02.832 INFO:tasks.cephadm.mon.smithi088:Stopping mon.smithi088... 2024-09-16T09:51:02.832 DEBUG:teuthology.orchestra.run.smithi088:> sudo systemctl stop ceph-a70c765e-740d-11ef-bceb-c7b262605968@mon.smithi088 2024-09-16T09:51:03.122 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:51:02 smithi088 systemd[1]: Stopping Ceph mon.smithi088 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:51:03.387 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:51:03 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi088[52754]: 2024-09-16T09:51:03.119+0000 7f3bba2d7640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi088 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T09:51:03.387 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:51:03 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi088[52754]: 2024-09-16T09:51:03.119+0000 7f3bba2d7640 -1 mon.smithi088@1(peon) e3 *** Got Signal Terminated *** 2024-09-16T09:51:03.663 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:51:03 smithi088 podman[71566]: 2024-09-16 09:51:03.386079035 +0000 UTC m=+0.358465825 container died 2e552147327ac311d346dc2bc8c6607195094064e66158a49b14e9748bd9b89c (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi088, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-16T09:51:04.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:51:03 smithi088 podman[71566]: 2024-09-16 09:51:03.870026707 +0000 UTC m=+0.842413484 container cleanup 2e552147327ac311d346dc2bc8c6607195094064e66158a49b14e9748bd9b89c (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi088, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True) 2024-09-16T09:51:04.051 INFO:journalctl@ceph.mon.smithi088.smithi088.stdout:Sep 16 09:51:03 smithi088 bash[71566]: ceph-a70c765e-740d-11ef-bceb-c7b262605968-mon-smithi088 2024-09-16T09:51:04.304 DEBUG:teuthology.orchestra.run.smithi088:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a70c765e-740d-11ef-bceb-c7b262605968@mon.smithi088.service' 2024-09-16T09:51:04.346 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-16T09:51:04.346 INFO:tasks.cephadm.mon.smithi088:Stopped mon.smithi088 2024-09-16T09:51:04.346 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-16T09:51:04.346 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.0 2024-09-16T09:51:04.683 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:51:04 smithi078 systemd[1]: Stopping Ceph osd.0 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:51:05.061 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:51:04 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0[68739]: 2024-09-16T09:51:04.681+0000 7fa1445ad640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T09:51:05.062 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:51:04 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0[68739]: 2024-09-16T09:51:04.681+0000 7fa1445ad640 -1 osd.0 163 *** Got signal Terminated *** 2024-09-16T09:51:05.062 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:51:04 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0[68739]: 2024-09-16T09:51:04.681+0000 7fa1445ad640 -1 osd.0 163 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T09:51:10.484 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:51:10 smithi078 podman[93941]: 2024-09-16 09:51:10.208426475 +0000 UTC m=+5.650230676 container died 85008b499cb7625e6b7c07fa49c39892daba63b94bdd83a62583dca70529aa14 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-16T09:51:10.814 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:51:10 smithi078 podman[93941]: 2024-09-16 09:51:10.572898252 +0000 UTC m=+6.014702423 container cleanup 85008b499cb7625e6b7c07fa49c39892daba63b94bdd83a62583dca70529aa14 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-16T09:51:10.814 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:51:10 smithi078 bash[93941]: ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0 2024-09-16T09:51:11.184 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:51:10 smithi078 podman[93956]: 2024-09-16 09:51:10.866285887 +0000 UTC m=+0.650058008 container remove 85008b499cb7625e6b7c07fa49c39892daba63b94bdd83a62583dca70529aa14 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-16T09:51:11.454 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:51:11 smithi078 podman[94045]: 2024-09-16 09:51:11.085997167 +0000 UTC m=+0.015753371 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:51:11.454 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:51:11 smithi078 podman[94045]: 2024-09-16 09:51:11.208986021 +0000 UTC m=+0.138742253 container create ab29b35787dc2ec3e1fc103754e18485ee21962eccd8aede3556767cb2130704 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-16T09:51:11.454 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:51:11 smithi078 podman[94045]: 2024-09-16 09:51:11.453252558 +0000 UTC m=+0.383008749 container init ab29b35787dc2ec3e1fc103754e18485ee21962eccd8aede3556767cb2130704 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-16T09:51:11.811 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:51:11 smithi078 podman[94045]: 2024-09-16 09:51:11.457044732 +0000 UTC m=+0.386800922 container start ab29b35787dc2ec3e1fc103754e18485ee21962eccd8aede3556767cb2130704 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-16T09:51:11.811 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:51:11 smithi078 podman[94045]: 2024-09-16 09:51:11.545091423 +0000 UTC m=+0.474847622 container attach ab29b35787dc2ec3e1fc103754e18485ee21962eccd8aede3556767cb2130704 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-16T09:51:11.811 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 16 09:51:11 smithi078 podman[94045]: 2024-09-16 09:51:11.645355052 +0000 UTC m=+0.575111249 container died ab29b35787dc2ec3e1fc103754e18485ee21962eccd8aede3556767cb2130704 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True) 2024-09-16T09:51:12.297 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.0.service' 2024-09-16T09:51:12.336 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-16T09:51:12.336 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-16T09:51:12.337 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-16T09:51:12.337 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.1 2024-09-16T09:51:12.561 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:51:12 smithi078 systemd[1]: Stopping Ceph osd.1 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:51:13.061 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:51:12 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1[71104]: 2024-09-16T09:51:12.680+0000 7f38d8b36640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T09:51:13.061 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:51:12 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1[71104]: 2024-09-16T09:51:12.680+0000 7f38d8b36640 -1 osd.1 163 *** Got signal Terminated *** 2024-09-16T09:51:13.061 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:51:12 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1[71104]: 2024-09-16T09:51:12.680+0000 7f38d8b36640 -1 osd.1 163 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T09:51:18.379 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:51:18 smithi078 podman[94148]: 2024-09-16 09:51:18.123250498 +0000 UTC m=+5.567537303 container died 12c137fcba709e979c75fe97d30e55655e30b8a73fbc369899cceccd1a5f90a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0) 2024-09-16T09:51:18.745 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:51:18 smithi078 podman[94148]: 2024-09-16 09:51:18.485296825 +0000 UTC m=+5.929583630 container cleanup 12c137fcba709e979c75fe97d30e55655e30b8a73fbc369899cceccd1a5f90a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-16T09:51:18.746 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:51:18 smithi078 bash[94148]: ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1 2024-09-16T09:51:19.060 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:51:18 smithi078 podman[94181]: 2024-09-16 09:51:18.744979142 +0000 UTC m=+0.621623998 container remove 12c137fcba709e979c75fe97d30e55655e30b8a73fbc369899cceccd1a5f90a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-16T09:51:19.311 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:51:19 smithi078 podman[94269]: 2024-09-16 09:51:18.962643386 +0000 UTC m=+0.016149990 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:51:19.312 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:51:19 smithi078 podman[94269]: 2024-09-16 09:51:19.070830284 +0000 UTC m=+0.124336881 container create e7ae18ed7bf7db84126ffa30b58f818c1b3cec597e70f624c74b06dccc485de7 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-16T09:51:19.312 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:51:19 smithi078 podman[94269]: 2024-09-16 09:51:19.29861181 +0000 UTC m=+0.352118413 container init e7ae18ed7bf7db84126ffa30b58f818c1b3cec597e70f624c74b06dccc485de7 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-16T09:51:19.312 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:51:19 smithi078 podman[94269]: 2024-09-16 09:51:19.302039231 +0000 UTC m=+0.355545830 container start e7ae18ed7bf7db84126ffa30b58f818c1b3cec597e70f624c74b06dccc485de7 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-16T09:51:19.778 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:51:19 smithi078 podman[94269]: 2024-09-16 09:51:19.398563703 +0000 UTC m=+0.452070310 container attach e7ae18ed7bf7db84126ffa30b58f818c1b3cec597e70f624c74b06dccc485de7 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-16T09:51:19.778 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 16 09:51:19 smithi078 podman[94269]: 2024-09-16 09:51:19.488796686 +0000 UTC m=+0.542303290 container died e7ae18ed7bf7db84126ffa30b58f818c1b3cec597e70f624c74b06dccc485de7 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-16T09:51:20.083 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.1.service' 2024-09-16T09:51:20.120 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-16T09:51:20.120 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-16T09:51:20.121 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-16T09:51:20.121 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.2 2024-09-16T09:51:20.440 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:51:20 smithi078 systemd[1]: Stopping Ceph osd.2 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:51:20.811 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:51:20 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2[73557]: 2024-09-16T09:51:20.438+0000 7f7457edc640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T09:51:20.811 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:51:20 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2[73557]: 2024-09-16T09:51:20.438+0000 7f7457edc640 -1 osd.2 163 *** Got signal Terminated *** 2024-09-16T09:51:20.811 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:51:20 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2[73557]: 2024-09-16T09:51:20.438+0000 7f7457edc640 -1 osd.2 163 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T09:51:26.224 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:51:25 smithi078 podman[94372]: 2024-09-16 09:51:25.941112918 +0000 UTC m=+5.601798717 container died 23a8b69c5c64a31a2d1ccd324b5bada8c6d962d9e11d5ca6fd78e325e3911275 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731) 2024-09-16T09:51:26.571 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:51:26 smithi078 podman[94372]: 2024-09-16 09:51:26.296647846 +0000 UTC m=+5.957333634 container cleanup 23a8b69c5c64a31a2d1ccd324b5bada8c6d962d9e11d5ca6fd78e325e3911275 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6) 2024-09-16T09:51:26.572 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:51:26 smithi078 bash[94372]: ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2 2024-09-16T09:51:26.811 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:51:26 smithi078 podman[94388]: 2024-09-16 09:51:26.556258702 +0000 UTC m=+0.618312383 container remove 23a8b69c5c64a31a2d1ccd324b5bada8c6d962d9e11d5ca6fd78e325e3911275 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True) 2024-09-16T09:51:27.202 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:51:26 smithi078 podman[94478]: 2024-09-16 09:51:26.777451558 +0000 UTC m=+0.016068152 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:51:27.202 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:51:26 smithi078 podman[94478]: 2024-09-16 09:51:26.890349086 +0000 UTC m=+0.128965680 container create 89c23a95c8b3c7f37eac017e478df6854a4959359959ded41f262e85f926f4c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-16T09:51:27.202 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:51:27 smithi078 podman[94478]: 2024-09-16 09:51:27.117979886 +0000 UTC m=+0.356596483 container init 89c23a95c8b3c7f37eac017e478df6854a4959359959ded41f262e85f926f4c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2-deactivate, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-16T09:51:27.202 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:51:27 smithi078 podman[94478]: 2024-09-16 09:51:27.121189068 +0000 UTC m=+0.359805670 container start 89c23a95c8b3c7f37eac017e478df6854a4959359959ded41f262e85f926f4c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-16T09:51:27.202 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:51:27 smithi078 podman[94478]: 2024-09-16 09:51:27.20108731 +0000 UTC m=+0.439703918 container attach 89c23a95c8b3c7f37eac017e478df6854a4959359959ded41f262e85f926f4c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-16T09:51:27.561 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 16 09:51:27 smithi078 podman[94478]: 2024-09-16 09:51:27.30796604 +0000 UTC m=+0.546582645 container died 89c23a95c8b3c7f37eac017e478df6854a4959359959ded41f262e85f926f4c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-2-deactivate, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-16T09:51:27.888 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.2.service' 2024-09-16T09:51:27.926 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-16T09:51:27.927 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-16T09:51:27.927 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-16T09:51:27.927 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.3 2024-09-16T09:51:28.243 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:51:28 smithi078 systemd[1]: Stopping Ceph osd.3 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:51:28.561 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:51:28 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3[75896]: 2024-09-16T09:51:28.241+0000 7f00f5177640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T09:51:28.561 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:51:28 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3[75896]: 2024-09-16T09:51:28.241+0000 7f00f5177640 -1 osd.3 163 *** Got signal Terminated *** 2024-09-16T09:51:28.561 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:51:28 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3[75896]: 2024-09-16T09:51:28.241+0000 7f00f5177640 -1 osd.3 163 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T09:51:31.051 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:30 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:30.654+0000 7fcc4bcfa640 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.392232+0000 front 2024-09-16T09:51:07.392273+0000 (oldest deadline 2024-09-16T09:51:30.292083+0000) 2024-09-16T09:51:31.311 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:51:30 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3[75896]: 2024-09-16T09:51:30.991+0000 7f00f178c640 -1 osd.3 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.125777+0000 front 2024-09-16T09:51:09.125838+0000 (oldest deadline 2024-09-16T09:51:30.225919+0000) 2024-09-16T09:51:31.538 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:31 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:31.183+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:51:31.801 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:31 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:31.676+0000 7fcc4bcfa640 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.392232+0000 front 2024-09-16T09:51:07.392273+0000 (oldest deadline 2024-09-16T09:51:30.292083+0000) 2024-09-16T09:51:31.801 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:31 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:31.536+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.940183+0000 front 2024-09-16T09:51:07.940161+0000 (oldest deadline 2024-09-16T09:51:31.440039+0000) 2024-09-16T09:51:32.296 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:51:32 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3[75896]: 2024-09-16T09:51:32.002+0000 7f00f178c640 -1 osd.3 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.125777+0000 front 2024-09-16T09:51:09.125838+0000 (oldest deadline 2024-09-16T09:51:30.225919+0000) 2024-09-16T09:51:32.300 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:32 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:32.139+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:51:32.801 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:32 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:32.711+0000 7fcc4bcfa640 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.392232+0000 front 2024-09-16T09:51:07.392273+0000 (oldest deadline 2024-09-16T09:51:30.292083+0000) 2024-09-16T09:51:32.801 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:32 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:32.508+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.940183+0000 front 2024-09-16T09:51:07.940161+0000 (oldest deadline 2024-09-16T09:51:31.440039+0000) 2024-09-16T09:51:33.311 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:51:33 smithi078 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3[75896]: 2024-09-16T09:51:32.999+0000 7f00f178c640 -1 osd.3 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.125777+0000 front 2024-09-16T09:51:09.125838+0000 (oldest deadline 2024-09-16T09:51:30.225919+0000) 2024-09-16T09:51:33.470 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:33 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:33.170+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:51:33.732 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:33 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:33.468+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.940183+0000 front 2024-09-16T09:51:07.940161+0000 (oldest deadline 2024-09-16T09:51:31.440039+0000) 2024-09-16T09:51:33.968 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:51:33 smithi078 podman[94579]: 2024-09-16 09:51:33.713057336 +0000 UTC m=+5.569286978 container died 00c9aa84d41d4002000786c60f19fe92d7e21fdd3e0fcd1dd7153959678f6634 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-16T09:51:34.051 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:33 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:33.729+0000 7fcc4bcfa640 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.392232+0000 front 2024-09-16T09:51:07.392273+0000 (oldest deadline 2024-09-16T09:51:30.292083+0000) 2024-09-16T09:51:34.260 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:51:34 smithi078 podman[94579]: 2024-09-16 09:51:34.049392872 +0000 UTC m=+5.905622513 container cleanup 00c9aa84d41d4002000786c60f19fe92d7e21fdd3e0fcd1dd7153959678f6634 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-16T09:51:34.260 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:51:34 smithi078 bash[94579]: ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3 2024-09-16T09:51:34.478 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:34 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:34.123+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:51:34.561 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:51:34 smithi078 podman[94612]: 2024-09-16 09:51:34.259453023 +0000 UTC m=+0.548583582 container remove 00c9aa84d41d4002000786c60f19fe92d7e21fdd3e0fcd1dd7153959678f6634 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS) 2024-09-16T09:51:34.801 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:34 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:34.476+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.940183+0000 front 2024-09-16T09:51:07.940161+0000 (oldest deadline 2024-09-16T09:51:31.440039+0000) 2024-09-16T09:51:34.801 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:34 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:34.726+0000 7fcc4bcfa640 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.392232+0000 front 2024-09-16T09:51:07.392273+0000 (oldest deadline 2024-09-16T09:51:30.292083+0000) 2024-09-16T09:51:34.947 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:51:34 smithi078 podman[94701]: 2024-09-16 09:51:34.562257108 +0000 UTC m=+0.016594116 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:51:34.947 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:51:34 smithi078 podman[94701]: 2024-09-16 09:51:34.668691043 +0000 UTC m=+0.123028094 container create 9bcb0caf71375962a34ead0f4dfda96bc992003922d9ff9302a14e9eeae201a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, io.buildah.version=1.37.2) 2024-09-16T09:51:34.947 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:51:34 smithi078 podman[94701]: 2024-09-16 09:51:34.870956314 +0000 UTC m=+0.325293324 container init 9bcb0caf71375962a34ead0f4dfda96bc992003922d9ff9302a14e9eeae201a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-16T09:51:34.947 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:51:34 smithi078 podman[94701]: 2024-09-16 09:51:34.874768263 +0000 UTC m=+0.329105279 container start 9bcb0caf71375962a34ead0f4dfda96bc992003922d9ff9302a14e9eeae201a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6) 2024-09-16T09:51:34.947 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:51:34 smithi078 podman[94701]: 2024-09-16 09:51:34.945913889 +0000 UTC m=+0.400250901 container attach 9bcb0caf71375962a34ead0f4dfda96bc992003922d9ff9302a14e9eeae201a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6) 2024-09-16T09:51:35.311 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 16 09:51:35 smithi078 podman[94701]: 2024-09-16 09:51:35.070309309 +0000 UTC m=+0.524646319 container died 9bcb0caf71375962a34ead0f4dfda96bc992003922d9ff9302a14e9eeae201a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-16T09:51:35.403 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:35 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:35.147+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:51:35.656 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.3.service' 2024-09-16T09:51:35.694 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-16T09:51:35.694 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-16T09:51:35.694 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-16T09:51:35.695 DEBUG:teuthology.orchestra.run.smithi088:> sudo systemctl stop ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.4 2024-09-16T09:51:35.709 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:35 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:35.474+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.940183+0000 front 2024-09-16T09:51:07.940161+0000 (oldest deadline 2024-09-16T09:51:31.440039+0000) 2024-09-16T09:51:35.710 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:51:35 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4[56635]: 2024-09-16T09:51:35.401+0000 7eff38277640 -1 osd.4 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.596052+0000 front 2024-09-16T09:51:09.596003+0000 (oldest deadline 2024-09-16T09:51:34.895963+0000) 2024-09-16T09:51:35.997 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:51:35 smithi088 systemd[1]: Stopping Ceph osd.4 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:51:35.997 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:35 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:35.707+0000 7fcc4bcfa640 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.392232+0000 front 2024-09-16T09:51:07.392273+0000 (oldest deadline 2024-09-16T09:51:30.292083+0000) 2024-09-16T09:51:36.301 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:36 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:36.190+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:51:36.301 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:51:35 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4[56635]: 2024-09-16T09:51:35.994+0000 7eff3bc62640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T09:51:36.301 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:51:35 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4[56635]: 2024-09-16T09:51:35.994+0000 7eff3bc62640 -1 osd.4 163 *** Got signal Terminated *** 2024-09-16T09:51:36.301 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:51:35 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4[56635]: 2024-09-16T09:51:35.994+0000 7eff3bc62640 -1 osd.4 163 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T09:51:36.714 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:36 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:36.493+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.940183+0000 front 2024-09-16T09:51:07.940161+0000 (oldest deadline 2024-09-16T09:51:31.440039+0000) 2024-09-16T09:51:36.715 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:51:36 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4[56635]: 2024-09-16T09:51:36.355+0000 7eff38277640 -1 osd.4 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.596052+0000 front 2024-09-16T09:51:09.596003+0000 (oldest deadline 2024-09-16T09:51:34.895963+0000) 2024-09-16T09:51:37.051 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:36 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:36.712+0000 7fcc4bcfa640 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.392232+0000 front 2024-09-16T09:51:07.392273+0000 (oldest deadline 2024-09-16T09:51:30.292083+0000) 2024-09-16T09:51:37.531 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:51:37 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4[56635]: 2024-09-16T09:51:37.338+0000 7eff38277640 -1 osd.4 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.596052+0000 front 2024-09-16T09:51:09.596003+0000 (oldest deadline 2024-09-16T09:51:34.895963+0000) 2024-09-16T09:51:37.531 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:37 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:37.172+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:51:37.801 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:37 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:37.734+0000 7fcc4bcfa640 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.392232+0000 front 2024-09-16T09:51:07.392273+0000 (oldest deadline 2024-09-16T09:51:30.292083+0000) 2024-09-16T09:51:37.801 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:37 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:37.528+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.940183+0000 front 2024-09-16T09:51:07.940161+0000 (oldest deadline 2024-09-16T09:51:31.440039+0000) 2024-09-16T09:51:38.515 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:38 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:38.175+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:51:38.515 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:51:38 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4[56635]: 2024-09-16T09:51:38.337+0000 7eff38277640 -1 osd.4 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.596052+0000 front 2024-09-16T09:51:09.596003+0000 (oldest deadline 2024-09-16T09:51:34.895963+0000) 2024-09-16T09:51:38.515 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:51:38 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4[56635]: 2024-09-16T09:51:38.337+0000 7eff38277640 -1 osd.4 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:15.396837+0000 front 2024-09-16T09:51:15.396851+0000 (oldest deadline 2024-09-16T09:51:38.296673+0000) 2024-09-16T09:51:38.778 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:38 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:38.513+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.940183+0000 front 2024-09-16T09:51:07.940161+0000 (oldest deadline 2024-09-16T09:51:31.440039+0000) 2024-09-16T09:51:39.051 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:38 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:38.775+0000 7fcc4bcfa640 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.392232+0000 front 2024-09-16T09:51:07.392273+0000 (oldest deadline 2024-09-16T09:51:30.292083+0000) 2024-09-16T09:51:39.487 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:39 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:39.160+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:51:39.487 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:51:39 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4[56635]: 2024-09-16T09:51:39.385+0000 7eff38277640 -1 osd.4 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.596052+0000 front 2024-09-16T09:51:09.596003+0000 (oldest deadline 2024-09-16T09:51:34.895963+0000) 2024-09-16T09:51:39.487 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:51:39 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4[56635]: 2024-09-16T09:51:39.385+0000 7eff38277640 -1 osd.4 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:15.396837+0000 front 2024-09-16T09:51:15.396851+0000 (oldest deadline 2024-09-16T09:51:38.296673+0000) 2024-09-16T09:51:39.763 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:39 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:39.485+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.940183+0000 front 2024-09-16T09:51:07.940161+0000 (oldest deadline 2024-09-16T09:51:31.440039+0000) 2024-09-16T09:51:40.051 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:39 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:39.761+0000 7fcc4bcfa640 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.392232+0000 front 2024-09-16T09:51:07.392273+0000 (oldest deadline 2024-09-16T09:51:30.292083+0000) 2024-09-16T09:51:40.051 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:39 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:39.761+0000 7fcc4bcfa640 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.993186+0000 front 2024-09-16T09:51:14.993223+0000 (oldest deadline 2024-09-16T09:51:39.093047+0000) 2024-09-16T09:51:40.526 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:51:40 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4[56635]: 2024-09-16T09:51:40.403+0000 7eff38277640 -1 osd.4 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.596052+0000 front 2024-09-16T09:51:09.596003+0000 (oldest deadline 2024-09-16T09:51:34.895963+0000) 2024-09-16T09:51:40.526 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:51:40 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4[56635]: 2024-09-16T09:51:40.403+0000 7eff38277640 -1 osd.4 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:15.396837+0000 front 2024-09-16T09:51:15.396851+0000 (oldest deadline 2024-09-16T09:51:38.296673+0000) 2024-09-16T09:51:40.526 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:40 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:40.162+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:51:40.526 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:40 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:40.162+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.164926+0000 front 2024-09-16T09:51:14.164761+0000 (oldest deadline 2024-09-16T09:51:39.464811+0000) 2024-09-16T09:51:40.801 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:40 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:40.524+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.940183+0000 front 2024-09-16T09:51:07.940161+0000 (oldest deadline 2024-09-16T09:51:31.440039+0000) 2024-09-16T09:51:41.202 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:40 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:40.807+0000 7fcc4bcfa640 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.392232+0000 front 2024-09-16T09:51:07.392273+0000 (oldest deadline 2024-09-16T09:51:30.292083+0000) 2024-09-16T09:51:41.202 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:40 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:40.807+0000 7fcc4bcfa640 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.993186+0000 front 2024-09-16T09:51:14.993223+0000 (oldest deadline 2024-09-16T09:51:39.093047+0000) 2024-09-16T09:51:41.526 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:41 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:41.200+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:51:41.526 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:41 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:41.200+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.164926+0000 front 2024-09-16T09:51:14.164761+0000 (oldest deadline 2024-09-16T09:51:39.464811+0000) 2024-09-16T09:51:41.798 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:51:41 smithi088 podman[71731]: 2024-09-16 09:51:41.524791197 +0000 UTC m=+5.637938539 container died 1d92e66b75d99e2f2c7aed2103af346bb772e9e20db20f0fd6e9f7e521fec78b (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-16T09:51:41.798 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:41 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:41.567+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.940183+0000 front 2024-09-16T09:51:07.940161+0000 (oldest deadline 2024-09-16T09:51:31.440039+0000) 2024-09-16T09:51:41.798 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:41 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:41.567+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:16.140758+0000 front 2024-09-16T09:51:16.140813+0000 (oldest deadline 2024-09-16T09:51:40.840751+0000) 2024-09-16T09:51:42.064 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:41 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:41.829+0000 7fcc4bcfa640 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.392232+0000 front 2024-09-16T09:51:07.392273+0000 (oldest deadline 2024-09-16T09:51:30.292083+0000) 2024-09-16T09:51:42.065 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:41 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:41.829+0000 7fcc4bcfa640 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.993186+0000 front 2024-09-16T09:51:14.993223+0000 (oldest deadline 2024-09-16T09:51:39.093047+0000) 2024-09-16T09:51:42.065 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:51:41 smithi088 podman[71731]: 2024-09-16 09:51:41.995751613 +0000 UTC m=+6.108898958 container cleanup 1d92e66b75d99e2f2c7aed2103af346bb772e9e20db20f0fd6e9f7e521fec78b (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-16T09:51:42.065 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:51:41 smithi088 bash[71731]: ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4 2024-09-16T09:51:42.544 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:51:42 smithi088 podman[71746]: 2024-09-16 09:51:42.40636975 +0000 UTC m=+0.872087705 container remove 1d92e66b75d99e2f2c7aed2103af346bb772e9e20db20f0fd6e9f7e521fec78b (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4, CEPH_POINT_RELEASE=, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-16T09:51:42.544 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:42 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:42.242+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:51:42.545 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:42 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:42.242+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.164926+0000 front 2024-09-16T09:51:14.164761+0000 (oldest deadline 2024-09-16T09:51:39.464811+0000) 2024-09-16T09:51:42.800 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:51:42 smithi088 podman[71834]: 2024-09-16 09:51:42.663428729 +0000 UTC m=+0.015742385 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:51:42.801 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:42 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:42.785+0000 7fcc4bcfa640 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.392232+0000 front 2024-09-16T09:51:07.392273+0000 (oldest deadline 2024-09-16T09:51:30.292083+0000) 2024-09-16T09:51:42.801 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:42 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:42.785+0000 7fcc4bcfa640 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.993186+0000 front 2024-09-16T09:51:14.993223+0000 (oldest deadline 2024-09-16T09:51:39.093047+0000) 2024-09-16T09:51:42.801 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:42 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:42.542+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.940183+0000 front 2024-09-16T09:51:07.940161+0000 (oldest deadline 2024-09-16T09:51:31.440039+0000) 2024-09-16T09:51:42.801 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:42 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:42.542+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:16.140758+0000 front 2024-09-16T09:51:16.140813+0000 (oldest deadline 2024-09-16T09:51:40.840751+0000) 2024-09-16T09:51:43.051 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:51:42 smithi088 podman[71834]: 2024-09-16 09:51:42.799008993 +0000 UTC m=+0.151322678 container create 004b5b2caab219472920c9644803eadbcf2cc2df40044c778b3174949dd85b94 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-16T09:51:43.527 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:43 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:43.291+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:51:43.527 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:43 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:43.291+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.164926+0000 front 2024-09-16T09:51:14.164761+0000 (oldest deadline 2024-09-16T09:51:39.464811+0000) 2024-09-16T09:51:43.528 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:51:43 smithi088 podman[71834]: 2024-09-16 09:51:43.060230753 +0000 UTC m=+0.412544397 container init 004b5b2caab219472920c9644803eadbcf2cc2df40044c778b3174949dd85b94 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-16T09:51:43.528 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:51:43 smithi088 podman[71834]: 2024-09-16 09:51:43.063660709 +0000 UTC m=+0.415974355 container start 004b5b2caab219472920c9644803eadbcf2cc2df40044c778b3174949dd85b94 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4-deactivate, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-16T09:51:43.528 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:51:43 smithi088 podman[71834]: 2024-09-16 09:51:43.151770665 +0000 UTC m=+0.504084309 container attach 004b5b2caab219472920c9644803eadbcf2cc2df40044c778b3174949dd85b94 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-16T09:51:43.528 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 16 09:51:43 smithi088 podman[71834]: 2024-09-16 09:51:43.247703282 +0000 UTC m=+0.600016934 container died 004b5b2caab219472920c9644803eadbcf2cc2df40044c778b3174949dd85b94 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-16T09:51:43.792 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:43 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:43.525+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.940183+0000 front 2024-09-16T09:51:07.940161+0000 (oldest deadline 2024-09-16T09:51:31.440039+0000) 2024-09-16T09:51:43.792 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:43 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:43.525+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:16.140758+0000 front 2024-09-16T09:51:16.140813+0000 (oldest deadline 2024-09-16T09:51:40.840751+0000) 2024-09-16T09:51:43.934 DEBUG:teuthology.orchestra.run.smithi088:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.4.service' 2024-09-16T09:51:44.013 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-16T09:51:44.013 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-16T09:51:44.014 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-16T09:51:44.014 DEBUG:teuthology.orchestra.run.smithi088:> sudo systemctl stop ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.5 2024-09-16T09:51:44.051 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:43 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:43.790+0000 7fcc4bcfa640 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.392232+0000 front 2024-09-16T09:51:07.392273+0000 (oldest deadline 2024-09-16T09:51:30.292083+0000) 2024-09-16T09:51:44.051 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:43 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:43.790+0000 7fcc4bcfa640 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.993186+0000 front 2024-09-16T09:51:14.993223+0000 (oldest deadline 2024-09-16T09:51:39.093047+0000) 2024-09-16T09:51:44.335 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:44 smithi088 systemd[1]: Stopping Ceph osd.5 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:51:44.335 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:44 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:44.317+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:51:44.335 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:44 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:44.317+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.164926+0000 front 2024-09-16T09:51:14.164761+0000 (oldest deadline 2024-09-16T09:51:39.464811+0000) 2024-09-16T09:51:44.784 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:44 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:44.548+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.940183+0000 front 2024-09-16T09:51:07.940161+0000 (oldest deadline 2024-09-16T09:51:31.440039+0000) 2024-09-16T09:51:44.784 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:44 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:44.548+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:16.140758+0000 front 2024-09-16T09:51:16.140813+0000 (oldest deadline 2024-09-16T09:51:40.840751+0000) 2024-09-16T09:51:44.785 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:44 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:44.333+0000 7fcc536ed640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T09:51:44.785 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:44 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:44.333+0000 7fcc536ed640 -1 osd.5 163 *** Got signal Terminated *** 2024-09-16T09:51:44.785 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:44 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:44.333+0000 7fcc536ed640 -1 osd.5 163 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T09:51:44.785 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:44 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:44.782+0000 7fcc4bcfa640 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.392232+0000 front 2024-09-16T09:51:07.392273+0000 (oldest deadline 2024-09-16T09:51:30.292083+0000) 2024-09-16T09:51:44.785 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:44 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:44.782+0000 7fcc4bcfa640 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.993186+0000 front 2024-09-16T09:51:14.993223+0000 (oldest deadline 2024-09-16T09:51:39.093047+0000) 2024-09-16T09:51:45.779 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:45 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:45.300+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:51:45.780 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:45 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:45.300+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.164926+0000 front 2024-09-16T09:51:14.164761+0000 (oldest deadline 2024-09-16T09:51:39.464811+0000) 2024-09-16T09:51:45.780 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:45 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:45.548+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.940183+0000 front 2024-09-16T09:51:07.940161+0000 (oldest deadline 2024-09-16T09:51:31.440039+0000) 2024-09-16T09:51:45.780 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:45 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:45.548+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:16.140758+0000 front 2024-09-16T09:51:16.140813+0000 (oldest deadline 2024-09-16T09:51:40.840751+0000) 2024-09-16T09:51:46.051 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:45 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:45.777+0000 7fcc4bcfa640 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.392232+0000 front 2024-09-16T09:51:07.392273+0000 (oldest deadline 2024-09-16T09:51:30.292083+0000) 2024-09-16T09:51:46.051 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:45 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:45.777+0000 7fcc4bcfa640 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.993186+0000 front 2024-09-16T09:51:14.993223+0000 (oldest deadline 2024-09-16T09:51:39.093047+0000) 2024-09-16T09:51:46.529 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:46 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:46.254+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:51:46.530 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:46 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:46.254+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.164926+0000 front 2024-09-16T09:51:14.164761+0000 (oldest deadline 2024-09-16T09:51:39.464811+0000) 2024-09-16T09:51:46.801 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:46 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:46.774+0000 7fcc4bcfa640 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.392232+0000 front 2024-09-16T09:51:07.392273+0000 (oldest deadline 2024-09-16T09:51:30.292083+0000) 2024-09-16T09:51:46.801 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:46 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:46.774+0000 7fcc4bcfa640 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.993186+0000 front 2024-09-16T09:51:14.993223+0000 (oldest deadline 2024-09-16T09:51:39.093047+0000) 2024-09-16T09:51:46.801 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:46 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:46.527+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.940183+0000 front 2024-09-16T09:51:07.940161+0000 (oldest deadline 2024-09-16T09:51:31.440039+0000) 2024-09-16T09:51:46.801 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:46 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:46.527+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:16.140758+0000 front 2024-09-16T09:51:16.140813+0000 (oldest deadline 2024-09-16T09:51:40.840751+0000) 2024-09-16T09:51:47.521 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:47 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:47.249+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:51:47.522 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:47 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:47.249+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.164926+0000 front 2024-09-16T09:51:14.164761+0000 (oldest deadline 2024-09-16T09:51:39.464811+0000) 2024-09-16T09:51:47.801 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:47 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:47.520+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.940183+0000 front 2024-09-16T09:51:07.940161+0000 (oldest deadline 2024-09-16T09:51:31.440039+0000) 2024-09-16T09:51:47.801 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:47 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:47.520+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:16.140758+0000 front 2024-09-16T09:51:16.140813+0000 (oldest deadline 2024-09-16T09:51:40.840751+0000) 2024-09-16T09:51:47.801 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:47 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:47.736+0000 7fcc4bcfa640 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.392232+0000 front 2024-09-16T09:51:07.392273+0000 (oldest deadline 2024-09-16T09:51:30.292083+0000) 2024-09-16T09:51:47.801 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:47 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:47.736+0000 7fcc4bcfa640 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.993186+0000 front 2024-09-16T09:51:14.993223+0000 (oldest deadline 2024-09-16T09:51:39.093047+0000) 2024-09-16T09:51:47.801 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:47 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:47.736+0000 7fcc4bcfa640 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-16T09:51:21.994069+0000 front 2024-09-16T09:51:21.994169+0000 (oldest deadline 2024-09-16T09:51:47.294134+0000) 2024-09-16T09:51:48.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:48 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:48.295+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:51:48.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:48 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:48.295+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.164926+0000 front 2024-09-16T09:51:14.164761+0000 (oldest deadline 2024-09-16T09:51:39.464811+0000) 2024-09-16T09:51:48.551 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:48 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:48.496+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.940183+0000 front 2024-09-16T09:51:07.940161+0000 (oldest deadline 2024-09-16T09:51:31.440039+0000) 2024-09-16T09:51:48.551 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:48 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:48.496+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:16.140758+0000 front 2024-09-16T09:51:16.140813+0000 (oldest deadline 2024-09-16T09:51:40.840751+0000) 2024-09-16T09:51:48.551 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:48 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:48.496+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-16T09:51:21.941606+0000 front 2024-09-16T09:51:21.941624+0000 (oldest deadline 2024-09-16T09:51:47.841485+0000) 2024-09-16T09:51:49.051 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:48 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:48.770+0000 7fcc4bcfa640 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.392232+0000 front 2024-09-16T09:51:07.392273+0000 (oldest deadline 2024-09-16T09:51:30.292083+0000) 2024-09-16T09:51:49.051 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:48 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:48.770+0000 7fcc4bcfa640 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.993186+0000 front 2024-09-16T09:51:14.993223+0000 (oldest deadline 2024-09-16T09:51:39.093047+0000) 2024-09-16T09:51:49.051 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:48 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5[58631]: 2024-09-16T09:51:48.770+0000 7fcc4bcfa640 -1 osd.5 163 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-16T09:51:21.994069+0000 front 2024-09-16T09:51:21.994169+0000 (oldest deadline 2024-09-16T09:51:47.294134+0000) 2024-09-16T09:51:49.527 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:49 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:49.257+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:51:49.527 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:49 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:49.257+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.164926+0000 front 2024-09-16T09:51:14.164761+0000 (oldest deadline 2024-09-16T09:51:39.464811+0000) 2024-09-16T09:51:49.801 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:49 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:49.525+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.940183+0000 front 2024-09-16T09:51:07.940161+0000 (oldest deadline 2024-09-16T09:51:31.440039+0000) 2024-09-16T09:51:49.801 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:49 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:49.525+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:16.140758+0000 front 2024-09-16T09:51:16.140813+0000 (oldest deadline 2024-09-16T09:51:40.840751+0000) 2024-09-16T09:51:49.801 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:49 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:49.525+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-16T09:51:21.941606+0000 front 2024-09-16T09:51:21.941624+0000 (oldest deadline 2024-09-16T09:51:47.841485+0000) 2024-09-16T09:51:50.161 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:49 smithi088 podman[71935]: 2024-09-16 09:51:49.905434931 +0000 UTC m=+5.676381491 container died 43abc1197fe24d7d85901200df15501ecccff3a180319873191e8a650f78810d (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-16T09:51:50.521 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:50 smithi088 podman[71935]: 2024-09-16 09:51:50.367066252 +0000 UTC m=+6.138012857 container cleanup 43abc1197fe24d7d85901200df15501ecccff3a180319873191e8a650f78810d (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, ceph=True, org.label-schema.license=GPLv2) 2024-09-16T09:51:50.521 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:50 smithi088 bash[71935]: ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5 2024-09-16T09:51:50.522 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:50 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:50.270+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:51:50.522 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:50 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:50.270+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.164926+0000 front 2024-09-16T09:51:14.164761+0000 (oldest deadline 2024-09-16T09:51:39.464811+0000) 2024-09-16T09:51:50.522 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:50 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:50.270+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-16T09:51:24.566802+0000 front 2024-09-16T09:51:24.566812+0000 (oldest deadline 2024-09-16T09:51:49.866739+0000) 2024-09-16T09:51:50.801 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:50 smithi088 podman[71951]: 2024-09-16 09:51:50.635161938 +0000 UTC m=+0.725156507 container remove 43abc1197fe24d7d85901200df15501ecccff3a180319873191e8a650f78810d (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.vendor=CentOS, ceph=True) 2024-09-16T09:51:50.801 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:50 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:50.519+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.940183+0000 front 2024-09-16T09:51:07.940161+0000 (oldest deadline 2024-09-16T09:51:31.440039+0000) 2024-09-16T09:51:50.801 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:50 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:50.519+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:16.140758+0000 front 2024-09-16T09:51:16.140813+0000 (oldest deadline 2024-09-16T09:51:40.840751+0000) 2024-09-16T09:51:50.801 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:50 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:50.519+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-16T09:51:21.941606+0000 front 2024-09-16T09:51:21.941624+0000 (oldest deadline 2024-09-16T09:51:47.841485+0000) 2024-09-16T09:51:51.239 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:50 smithi088 podman[72038]: 2024-09-16 09:51:50.881516846 +0000 UTC m=+0.015808810 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:51:51.239 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:50 smithi088 podman[72038]: 2024-09-16 09:51:50.994155802 +0000 UTC m=+0.128447750 container create fabd634a51890eef52ba1d82507c846ce623eb0768179b096deca65f090f101e (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-16T09:51:51.239 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:51 smithi088 podman[72038]: 2024-09-16 09:51:51.238366273 +0000 UTC m=+0.372658227 container init fabd634a51890eef52ba1d82507c846ce623eb0768179b096deca65f090f101e (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-16T09:51:51.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:51 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:51.310+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:51:51.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:51 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:51.310+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.164926+0000 front 2024-09-16T09:51:14.164761+0000 (oldest deadline 2024-09-16T09:51:39.464811+0000) 2024-09-16T09:51:51.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:51 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:51.310+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-16T09:51:24.566802+0000 front 2024-09-16T09:51:24.566812+0000 (oldest deadline 2024-09-16T09:51:49.866739+0000) 2024-09-16T09:51:51.551 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:51 smithi088 podman[72038]: 2024-09-16 09:51:51.241834287 +0000 UTC m=+0.376126236 container start fabd634a51890eef52ba1d82507c846ce623eb0768179b096deca65f090f101e (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5-deactivate, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True) 2024-09-16T09:51:51.551 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:51 smithi088 podman[72038]: 2024-09-16 09:51:51.321546763 +0000 UTC m=+0.455838713 container attach fabd634a51890eef52ba1d82507c846ce623eb0768179b096deca65f090f101e (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-16T09:51:51.551 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 16 09:51:51 smithi088 podman[72038]: 2024-09-16 09:51:51.427218732 +0000 UTC m=+0.561510682 container died fabd634a51890eef52ba1d82507c846ce623eb0768179b096deca65f090f101e (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-16T09:51:52.001 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:51 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:51.552+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.940183+0000 front 2024-09-16T09:51:07.940161+0000 (oldest deadline 2024-09-16T09:51:31.440039+0000) 2024-09-16T09:51:52.001 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:51 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:51.552+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:16.140758+0000 front 2024-09-16T09:51:16.140813+0000 (oldest deadline 2024-09-16T09:51:40.840751+0000) 2024-09-16T09:51:52.001 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:51 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:51.552+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-16T09:51:21.941606+0000 front 2024-09-16T09:51:21.941624+0000 (oldest deadline 2024-09-16T09:51:47.841485+0000) 2024-09-16T09:51:52.019 DEBUG:teuthology.orchestra.run.smithi088:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.5.service' 2024-09-16T09:51:52.057 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-16T09:51:52.057 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-16T09:51:52.058 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-16T09:51:52.058 DEBUG:teuthology.orchestra.run.smithi088:> sudo systemctl stop ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.6 2024-09-16T09:51:52.301 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:52 smithi088 systemd[1]: Stopping Ceph osd.6 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:51:52.581 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:52 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:52.411+0000 7fc16d66e640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T09:51:52.581 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:52 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:52.411+0000 7fc16d66e640 -1 osd.6 163 *** Got signal Terminated *** 2024-09-16T09:51:52.581 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:52 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:52.411+0000 7fc16d66e640 -1 osd.6 163 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T09:51:52.581 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:52 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:52.579+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.940183+0000 front 2024-09-16T09:51:07.940161+0000 (oldest deadline 2024-09-16T09:51:31.440039+0000) 2024-09-16T09:51:52.581 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:52 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:52.321+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:51:52.582 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:52 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:52.321+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.164926+0000 front 2024-09-16T09:51:14.164761+0000 (oldest deadline 2024-09-16T09:51:39.464811+0000) 2024-09-16T09:51:52.582 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:52 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:52.321+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-16T09:51:24.566802+0000 front 2024-09-16T09:51:24.566812+0000 (oldest deadline 2024-09-16T09:51:49.866739+0000) 2024-09-16T09:51:53.051 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:52 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:52.579+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:16.140758+0000 front 2024-09-16T09:51:16.140813+0000 (oldest deadline 2024-09-16T09:51:40.840751+0000) 2024-09-16T09:51:53.051 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:52 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:52.579+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-16T09:51:21.941606+0000 front 2024-09-16T09:51:21.941624+0000 (oldest deadline 2024-09-16T09:51:47.841485+0000) 2024-09-16T09:51:53.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:53 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:53.291+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:51:53.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:53 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:53.291+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.164926+0000 front 2024-09-16T09:51:14.164761+0000 (oldest deadline 2024-09-16T09:51:39.464811+0000) 2024-09-16T09:51:53.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:53 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:53.291+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-16T09:51:24.566802+0000 front 2024-09-16T09:51:24.566812+0000 (oldest deadline 2024-09-16T09:51:49.866739+0000) 2024-09-16T09:51:54.051 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:53 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:53.601+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.940183+0000 front 2024-09-16T09:51:07.940161+0000 (oldest deadline 2024-09-16T09:51:31.440039+0000) 2024-09-16T09:51:54.051 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:53 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:53.601+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:16.140758+0000 front 2024-09-16T09:51:16.140813+0000 (oldest deadline 2024-09-16T09:51:40.840751+0000) 2024-09-16T09:51:54.051 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:53 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:53.601+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-16T09:51:21.941606+0000 front 2024-09-16T09:51:21.941624+0000 (oldest deadline 2024-09-16T09:51:47.841485+0000) 2024-09-16T09:51:54.587 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:54 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:54.316+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:51:54.587 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:54 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:54.316+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.164926+0000 front 2024-09-16T09:51:14.164761+0000 (oldest deadline 2024-09-16T09:51:39.464811+0000) 2024-09-16T09:51:54.587 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:54 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:54.316+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-16T09:51:24.566802+0000 front 2024-09-16T09:51:24.566812+0000 (oldest deadline 2024-09-16T09:51:49.866739+0000) 2024-09-16T09:51:55.051 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:54 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:54.585+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.940183+0000 front 2024-09-16T09:51:07.940161+0000 (oldest deadline 2024-09-16T09:51:31.440039+0000) 2024-09-16T09:51:55.051 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:54 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:54.585+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:16.140758+0000 front 2024-09-16T09:51:16.140813+0000 (oldest deadline 2024-09-16T09:51:40.840751+0000) 2024-09-16T09:51:55.051 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:54 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:54.585+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-16T09:51:21.941606+0000 front 2024-09-16T09:51:21.941624+0000 (oldest deadline 2024-09-16T09:51:47.841485+0000) 2024-09-16T09:51:55.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:55 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:55.288+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:51:55.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:55 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:55.288+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.164926+0000 front 2024-09-16T09:51:14.164761+0000 (oldest deadline 2024-09-16T09:51:39.464811+0000) 2024-09-16T09:51:55.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:55 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:55.288+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-16T09:51:24.566802+0000 front 2024-09-16T09:51:24.566812+0000 (oldest deadline 2024-09-16T09:51:49.866739+0000) 2024-09-16T09:51:55.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:55 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:55.288+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-16T09:51:32.167765+0000 front 2024-09-16T09:51:32.167773+0000 (oldest deadline 2024-09-16T09:51:54.467719+0000) 2024-09-16T09:51:56.053 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:55 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:55.581+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.940183+0000 front 2024-09-16T09:51:07.940161+0000 (oldest deadline 2024-09-16T09:51:31.440039+0000) 2024-09-16T09:51:56.053 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:55 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:55.581+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:16.140758+0000 front 2024-09-16T09:51:16.140813+0000 (oldest deadline 2024-09-16T09:51:40.840751+0000) 2024-09-16T09:51:56.053 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:55 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:55.581+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-16T09:51:21.941606+0000 front 2024-09-16T09:51:21.941624+0000 (oldest deadline 2024-09-16T09:51:47.841485+0000) 2024-09-16T09:51:56.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:56 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:56.271+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:51:56.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:56 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:56.271+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.164926+0000 front 2024-09-16T09:51:14.164761+0000 (oldest deadline 2024-09-16T09:51:39.464811+0000) 2024-09-16T09:51:56.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:56 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:56.271+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-16T09:51:24.566802+0000 front 2024-09-16T09:51:24.566812+0000 (oldest deadline 2024-09-16T09:51:49.866739+0000) 2024-09-16T09:51:56.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:56 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:56.271+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-16T09:51:32.167765+0000 front 2024-09-16T09:51:32.167773+0000 (oldest deadline 2024-09-16T09:51:54.467719+0000) 2024-09-16T09:51:57.051 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:56 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:56.595+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:07.940183+0000 front 2024-09-16T09:51:07.940161+0000 (oldest deadline 2024-09-16T09:51:31.440039+0000) 2024-09-16T09:51:57.051 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:56 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:56.595+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:16.140758+0000 front 2024-09-16T09:51:16.140813+0000 (oldest deadline 2024-09-16T09:51:40.840751+0000) 2024-09-16T09:51:57.051 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:56 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:56.595+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-16T09:51:21.941606+0000 front 2024-09-16T09:51:21.941624+0000 (oldest deadline 2024-09-16T09:51:47.841485+0000) 2024-09-16T09:51:57.051 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:56 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6[60670]: 2024-09-16T09:51:56.595+0000 7fc169c83640 -1 osd.6 163 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-16T09:51:31.842680+0000 front 2024-09-16T09:51:31.842653+0000 (oldest deadline 2024-09-16T09:51:56.542641+0000) 2024-09-16T09:51:57.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:57 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:57.288+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:51:57.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:57 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:57.288+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.164926+0000 front 2024-09-16T09:51:14.164761+0000 (oldest deadline 2024-09-16T09:51:39.464811+0000) 2024-09-16T09:51:57.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:57 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:57.288+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-16T09:51:24.566802+0000 front 2024-09-16T09:51:24.566812+0000 (oldest deadline 2024-09-16T09:51:49.866739+0000) 2024-09-16T09:51:57.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:57 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:57.288+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-16T09:51:32.167765+0000 front 2024-09-16T09:51:32.167773+0000 (oldest deadline 2024-09-16T09:51:54.467719+0000) 2024-09-16T09:51:58.180 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:57 smithi088 podman[72141]: 2024-09-16 09:51:57.84045002 +0000 UTC m=+5.561048574 container died 31302f8f7d8fa187cf12027dd840736a144a86b3235f1a4acf89fc453b5e51c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-16T09:51:58.551 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:58 smithi088 podman[72141]: 2024-09-16 09:51:58.353287264 +0000 UTC m=+6.073885848 container cleanup 31302f8f7d8fa187cf12027dd840736a144a86b3235f1a4acf89fc453b5e51c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-16T09:51:58.552 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:58 smithi088 bash[72141]: ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6 2024-09-16T09:51:58.552 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:58 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:58.326+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:51:58.552 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:58 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:58.326+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.164926+0000 front 2024-09-16T09:51:14.164761+0000 (oldest deadline 2024-09-16T09:51:39.464811+0000) 2024-09-16T09:51:58.552 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:58 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:58.326+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-16T09:51:24.566802+0000 front 2024-09-16T09:51:24.566812+0000 (oldest deadline 2024-09-16T09:51:49.866739+0000) 2024-09-16T09:51:58.552 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:58 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:58.326+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-16T09:51:32.167765+0000 front 2024-09-16T09:51:32.167773+0000 (oldest deadline 2024-09-16T09:51:54.467719+0000) 2024-09-16T09:51:58.981 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:58 smithi088 podman[72173]: 2024-09-16 09:51:58.580264913 +0000 UTC m=+0.731677975 container remove 31302f8f7d8fa187cf12027dd840736a144a86b3235f1a4acf89fc453b5e51c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-16T09:51:59.284 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:58 smithi088 podman[72259]: 2024-09-16 09:51:58.882737898 +0000 UTC m=+0.015705277 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:51:59.285 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:58 smithi088 podman[72259]: 2024-09-16 09:51:58.997428835 +0000 UTC m=+0.130396203 container create 7196acc41cfe03e207b191a1baa5bf44098473608cbb0a6401623c1e31842b19 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6-deactivate, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-16T09:51:59.285 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:59 smithi088 podman[72259]: 2024-09-16 09:51:59.224791197 +0000 UTC m=+0.357758567 container init 7196acc41cfe03e207b191a1baa5bf44098473608cbb0a6401623c1e31842b19 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6) 2024-09-16T09:51:59.285 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:59 smithi088 podman[72259]: 2024-09-16 09:51:59.228129902 +0000 UTC m=+0.361097280 container start 7196acc41cfe03e207b191a1baa5bf44098473608cbb0a6401623c1e31842b19 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-16T09:51:59.551 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:59 smithi088 podman[72259]: 2024-09-16 09:51:59.325050743 +0000 UTC m=+0.458018121 container attach 7196acc41cfe03e207b191a1baa5bf44098473608cbb0a6401623c1e31842b19 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-16T09:51:59.551 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 16 09:51:59 smithi088 podman[72259]: 2024-09-16 09:51:59.420768701 +0000 UTC m=+0.553736073 container died 7196acc41cfe03e207b191a1baa5bf44098473608cbb0a6401623c1e31842b19 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-6-deactivate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-16T09:51:59.552 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:59 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:59.282+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:51:59.552 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:59 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:59.282+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.164926+0000 front 2024-09-16T09:51:14.164761+0000 (oldest deadline 2024-09-16T09:51:39.464811+0000) 2024-09-16T09:51:59.552 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:59 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:59.282+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-16T09:51:24.566802+0000 front 2024-09-16T09:51:24.566812+0000 (oldest deadline 2024-09-16T09:51:49.866739+0000) 2024-09-16T09:51:59.552 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:51:59 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:51:59.282+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-16T09:51:32.167765+0000 front 2024-09-16T09:51:32.167773+0000 (oldest deadline 2024-09-16T09:51:54.467719+0000) 2024-09-16T09:52:00.037 DEBUG:teuthology.orchestra.run.smithi088:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.6.service' 2024-09-16T09:52:00.074 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-16T09:52:00.074 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-16T09:52:00.074 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-16T09:52:00.075 DEBUG:teuthology.orchestra.run.smithi088:> sudo systemctl stop ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.7 2024-09-16T09:52:00.301 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:00 smithi088 systemd[1]: Stopping Ceph osd.7 for a70c765e-740d-11ef-bceb-c7b262605968... 2024-09-16T09:52:00.301 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:00 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:52:00.234+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:52:00.301 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:00 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:52:00.234+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.164926+0000 front 2024-09-16T09:51:14.164761+0000 (oldest deadline 2024-09-16T09:51:39.464811+0000) 2024-09-16T09:52:00.302 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:00 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:52:00.234+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-16T09:51:24.566802+0000 front 2024-09-16T09:51:24.566812+0000 (oldest deadline 2024-09-16T09:51:49.866739+0000) 2024-09-16T09:52:00.302 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:00 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:52:00.234+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-16T09:51:32.167765+0000 front 2024-09-16T09:51:32.167773+0000 (oldest deadline 2024-09-16T09:51:54.467719+0000) 2024-09-16T09:52:00.801 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:00 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:52:00.397+0000 7ff78e714640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T09:52:00.801 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:00 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:52:00.397+0000 7ff78e714640 -1 osd.7 163 *** Got signal Terminated *** 2024-09-16T09:52:00.801 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:00 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:52:00.397+0000 7ff78e714640 -1 osd.7 163 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-16T09:52:01.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:01 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:52:01.276+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:52:01.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:01 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:52:01.276+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.164926+0000 front 2024-09-16T09:51:14.164761+0000 (oldest deadline 2024-09-16T09:51:39.464811+0000) 2024-09-16T09:52:01.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:01 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:52:01.276+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-16T09:51:24.566802+0000 front 2024-09-16T09:51:24.566812+0000 (oldest deadline 2024-09-16T09:51:49.866739+0000) 2024-09-16T09:52:01.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:01 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:52:01.276+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-16T09:51:32.167765+0000 front 2024-09-16T09:51:32.167773+0000 (oldest deadline 2024-09-16T09:51:54.467719+0000) 2024-09-16T09:52:02.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:02 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:52:02.252+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:52:02.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:02 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:52:02.252+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.164926+0000 front 2024-09-16T09:51:14.164761+0000 (oldest deadline 2024-09-16T09:51:39.464811+0000) 2024-09-16T09:52:02.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:02 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:52:02.252+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-16T09:51:24.566802+0000 front 2024-09-16T09:51:24.566812+0000 (oldest deadline 2024-09-16T09:51:49.866739+0000) 2024-09-16T09:52:02.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:02 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:52:02.252+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-16T09:51:32.167765+0000 front 2024-09-16T09:51:32.167773+0000 (oldest deadline 2024-09-16T09:51:54.467719+0000) 2024-09-16T09:52:02.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:02 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:52:02.252+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.88:6804 osd.4 since back 2024-09-16T09:51:36.168580+0000 front 2024-09-16T09:51:36.168523+0000 (oldest deadline 2024-09-16T09:52:02.068512+0000) 2024-09-16T09:52:03.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:03 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:52:03.254+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:52:03.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:03 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:52:03.254+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.164926+0000 front 2024-09-16T09:51:14.164761+0000 (oldest deadline 2024-09-16T09:51:39.464811+0000) 2024-09-16T09:52:03.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:03 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:52:03.254+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-16T09:51:24.566802+0000 front 2024-09-16T09:51:24.566812+0000 (oldest deadline 2024-09-16T09:51:49.866739+0000) 2024-09-16T09:52:03.552 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:03 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:52:03.254+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-16T09:51:32.167765+0000 front 2024-09-16T09:51:32.167773+0000 (oldest deadline 2024-09-16T09:51:54.467719+0000) 2024-09-16T09:52:03.552 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:03 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:52:03.254+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.88:6804 osd.4 since back 2024-09-16T09:51:36.168580+0000 front 2024-09-16T09:51:36.168523+0000 (oldest deadline 2024-09-16T09:52:02.068512+0000) 2024-09-16T09:52:04.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:04 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:52:04.289+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:52:04.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:04 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:52:04.289+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.164926+0000 front 2024-09-16T09:51:14.164761+0000 (oldest deadline 2024-09-16T09:51:39.464811+0000) 2024-09-16T09:52:04.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:04 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:52:04.289+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-16T09:51:24.566802+0000 front 2024-09-16T09:51:24.566812+0000 (oldest deadline 2024-09-16T09:51:49.866739+0000) 2024-09-16T09:52:04.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:04 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:52:04.289+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-16T09:51:32.167765+0000 front 2024-09-16T09:51:32.167773+0000 (oldest deadline 2024-09-16T09:51:54.467719+0000) 2024-09-16T09:52:04.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:04 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:52:04.289+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.88:6804 osd.4 since back 2024-09-16T09:51:36.168580+0000 front 2024-09-16T09:51:36.168523+0000 (oldest deadline 2024-09-16T09:52:02.068512+0000) 2024-09-16T09:52:05.801 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:05 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:52:05.329+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-16T09:51:09.163113+0000 front 2024-09-16T09:51:09.163099+0000 (oldest deadline 2024-09-16T09:51:30.262951+0000) 2024-09-16T09:52:05.801 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:05 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:52:05.329+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-16T09:51:14.164926+0000 front 2024-09-16T09:51:14.164761+0000 (oldest deadline 2024-09-16T09:51:39.464811+0000) 2024-09-16T09:52:05.801 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:05 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:52:05.329+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-16T09:51:24.566802+0000 front 2024-09-16T09:51:24.566812+0000 (oldest deadline 2024-09-16T09:51:49.866739+0000) 2024-09-16T09:52:05.801 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:05 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:52:05.329+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-16T09:51:32.167765+0000 front 2024-09-16T09:51:32.167773+0000 (oldest deadline 2024-09-16T09:51:54.467719+0000) 2024-09-16T09:52:05.802 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:05 smithi088 ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7[62688]: 2024-09-16T09:52:05.329+0000 7ff786d21640 -1 osd.7 163 heartbeat_check: no reply from 172.21.15.88:6804 osd.4 since back 2024-09-16T09:51:36.168580+0000 front 2024-09-16T09:51:36.168523+0000 (oldest deadline 2024-09-16T09:52:02.068512+0000) 2024-09-16T09:52:06.233 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:05 smithi088 podman[72362]: 2024-09-16 09:52:05.962215125 +0000 UTC m=+5.657006244 container died f3768fef8040c16c038f02f715c680dd3c64556d3095dd4b785c6ef996fe3091 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=) 2024-09-16T09:52:06.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:06 smithi088 podman[72362]: 2024-09-16 09:52:06.330719495 +0000 UTC m=+6.025510612 container cleanup f3768fef8040c16c038f02f715c680dd3c64556d3095dd4b785c6ef996fe3091 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6) 2024-09-16T09:52:06.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:06 smithi088 bash[72362]: ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7 2024-09-16T09:52:06.888 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:06 smithi088 podman[72389]: 2024-09-16 09:52:06.578547568 +0000 UTC m=+0.616964516 container remove f3768fef8040c16c038f02f715c680dd3c64556d3095dd4b785c6ef996fe3091 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-16T09:52:07.152 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:06 smithi088 podman[72478]: 2024-09-16 09:52:06.790424545 +0000 UTC m=+0.015699778 image pull 261f2dc7b20f078cba20226e1a82a867f88de1a71ca34322497f8520cc32e62e quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f 2024-09-16T09:52:07.152 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:06 smithi088 podman[72478]: 2024-09-16 09:52:06.899186414 +0000 UTC m=+0.124461648 container create ff2787a0963382b53bc6bab6c1a21fa571a64451b5563d46705c4e91fb1eb183 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-16T09:52:07.152 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:07 smithi088 podman[72478]: 2024-09-16 09:52:07.151450921 +0000 UTC m=+0.376726155 container init ff2787a0963382b53bc6bab6c1a21fa571a64451b5563d46705c4e91fb1eb183 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-16T09:52:07.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:07 smithi088 podman[72478]: 2024-09-16 09:52:07.154807746 +0000 UTC m=+0.380082980 container start ff2787a0963382b53bc6bab6c1a21fa571a64451b5563d46705c4e91fb1eb183 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-16T09:52:07.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:07 smithi088 podman[72478]: 2024-09-16 09:52:07.243290342 +0000 UTC m=+0.468565591 container attach ff2787a0963382b53bc6bab6c1a21fa571a64451b5563d46705c4e91fb1eb183 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-16T09:52:07.551 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 16 09:52:07 smithi088 podman[72478]: 2024-09-16 09:52:07.349308183 +0000 UTC m=+0.574583429 container died ff2787a0963382b53bc6bab6c1a21fa571a64451b5563d46705c4e91fb1eb183 (image=quay.ceph.io/ceph-ci/ceph@sha256:2dbeb26ef01ea72be3b27ac61f923de34ec1a7d8b2fcd428841fc646734c133f, name=ceph-a70c765e-740d-11ef-bceb-c7b262605968-osd-7-deactivate, ceph=True, RELEASE=wip-athakkar-testing-2024-09-12-1857-66e33d6, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-16T09:52:07.945 DEBUG:teuthology.orchestra.run.smithi088:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a70c765e-740d-11ef-bceb-c7b262605968@osd.7.service' 2024-09-16T09:52:07.986 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-16T09:52:07.986 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-16T09:52:07.986 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid a70c765e-740d-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-16T09:55:06.158 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid a70c765e-740d-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-16T09:55:27.790 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-16T09:55:27.827 DEBUG:teuthology.orchestra.run.smithi088:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-16T09:55:27.854 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-16T09:55:27.856 DEBUG:teuthology.misc:Transferring archived files from smithi078:/var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/avan-2024-09-16_08:23:53-orch:cephadm-wip-athakkar-testing-2024-09-12-1857-distro-default-smithi/7907040/remote/smithi078/crash 2024-09-16T09:55:27.857 DEBUG:teuthology.orchestra.run.smithi078:> sudo tar c -f - -C /var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/crash -- . 2024-09-16T09:55:27.910 INFO:teuthology.orchestra.run.smithi078.stderr:tar: /var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-16T09:55:27.910 INFO:teuthology.orchestra.run.smithi078.stderr:tar: Error is not recoverable: exiting now 2024-09-16T09:55:27.912 DEBUG:teuthology.misc:Transferring archived files from smithi088:/var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/avan-2024-09-16_08:23:53-orch:cephadm-wip-athakkar-testing-2024-09-12-1857-distro-default-smithi/7907040/remote/smithi088/crash 2024-09-16T09:55:27.912 DEBUG:teuthology.orchestra.run.smithi088:> sudo tar c -f - -C /var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/crash -- . 2024-09-16T09:55:27.950 INFO:teuthology.orchestra.run.smithi088.stderr:tar: /var/lib/ceph/a70c765e-740d-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-16T09:55:27.950 INFO:teuthology.orchestra.run.smithi088.stderr:tar: Error is not recoverable: exiting now 2024-09-16T09:55:27.951 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-16T09:55:27.952 DEBUG:teuthology.orchestra.run.smithi078:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v CEPHADM_REFRESH_FAILED | head -n 1 2024-09-16T09:55:27.980 INFO:tasks.cephadm:Compressing logs... 2024-09-16T09:55:27.980 DEBUG:teuthology.orchestra.run.smithi078:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-16T09:55:28.023 DEBUG:teuthology.orchestra.run.smithi088:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-16T09:55:28.045 INFO:teuthology.orchestra.run.smithi078.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-osd.0.log 2024-09-16T09:55:28.046 INFO:teuthology.orchestra.run.smithi078.stderr:: No such file or directory 2024-09-16T09:55:28.046 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-osd.1.log 2024-09-16T09:55:28.048 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-16T09:55:28.048 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-osd.0.log: /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi078.hiotnn.log 2024-09-16T09:55:28.049 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-mon.smithi078.log 2024-09-16T09:55:28.049 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-16T09:55:28.049 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi078.hiotnn.log: /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-mon.smithi078.log: gzip -5 --verbose -- /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-mgr.smithi078.rrfxge.log 2024-09-16T09:55:28.049 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-16T09:55:28.049 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-mgr.smithi078.rrfxge.log: gzip -5 --verbose -- /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-osd.2.log 2024-09-16T09:55:28.050 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph.cephadm.log: 85.8% -- replaced with /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-16T09:55:28.050 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph.log 2024-09-16T09:55:28.051 INFO:teuthology.orchestra.run.smithi088.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-mon.smithi088.log 2024-09-16T09:55:28.051 INFO:teuthology.orchestra.run.smithi088.stderr:: No such file or directory 2024-09-16T09:55:28.051 INFO:teuthology.orchestra.run.smithi088.stderr:gzip -5 --verbose -- /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-osd.7.log 2024-09-16T09:55:28.051 INFO:teuthology.orchestra.run.smithi088.stderr:gzip -5 --verbose -- /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-16T09:55:28.051 INFO:teuthology.orchestra.run.smithi088.stderr:/var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-mon.smithi088.log: /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-osd.4.log 2024-09-16T09:55:28.055 INFO:teuthology.orchestra.run.smithi088.stderr:/var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi088.dbunob.log 2024-09-16T09:55:28.055 INFO:teuthology.orchestra.run.smithi088.stderr:/var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-16T09:55:28.055 INFO:teuthology.orchestra.run.smithi088.stderr:/var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi088.dbunob.log: gzip -5 --verbose -- /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-16T09:55:28.055 INFO:teuthology.orchestra.run.smithi088.stderr:/var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 87.6% -- replaced with /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi088.dbunob.log.gz 2024-09-16T09:55:28.055 INFO:teuthology.orchestra.run.smithi088.stderr: --verbose -- /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-mgr.smithi088.ncxfdz.log 2024-09-16T09:55:28.055 INFO:teuthology.orchestra.run.smithi088.stderr:/var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-osd.6.log 2024-09-16T09:55:28.055 INFO:teuthology.orchestra.run.smithi088.stderr:/var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-mgr.smithi088.ncxfdz.log: gzip -5 --verbose -- /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-osd.5.log 2024-09-16T09:55:28.055 INFO:teuthology.orchestra.run.smithi088.stderr: 85.7% -- replaced with /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-16T09:55:28.056 INFO:teuthology.orchestra.run.smithi088.stderr:/var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-osd.6.log: 91.6% -- replaced with /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-16T09:55:28.056 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-osd.2.log: 91.4% -- replaced with /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-16T09:55:28.058 INFO:teuthology.orchestra.run.smithi088.stderr:gzip -5 --verbose -- /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph.log 2024-09-16T09:55:28.058 INFO:teuthology.orchestra.run.smithi088.stderr:/var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-16T09:55:28.059 INFO:teuthology.orchestra.run.smithi088.stderr:/var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph.log: 91.2% -- replaced with /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-16T09:55:28.060 INFO:teuthology.orchestra.run.smithi078.stderr: 91.1% -- replaced with /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-16T09:55:28.061 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-osd.3.log 2024-09-16T09:55:28.062 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-16T09:55:28.062 INFO:teuthology.orchestra.run.smithi088.stderr:/var/log/ceph/cephadm.log: 87.0% -- replaced with /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-16T09:55:28.065 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-osd.3.log: 87.1% -- replaced with /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-16T09:55:28.068 INFO:teuthology.orchestra.run.smithi088.stderr: 91.0% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-16T09:55:28.071 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cephadm.log: 91.2% -- replaced with /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi078.hiotnn.log.gz 2024-09-16T09:55:28.105 INFO:teuthology.orchestra.run.smithi078.stderr: 91.7% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-16T09:55:28.119 INFO:teuthology.orchestra.run.smithi088.stderr: 89.9% -- replaced with /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-mgr.smithi088.ncxfdz.log.gz 2024-09-16T09:55:28.357 INFO:teuthology.orchestra.run.smithi088.stderr: 92.6% -- replaced with /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-mon.smithi088.log.gz 2024-09-16T09:55:28.430 INFO:teuthology.orchestra.run.smithi078.stderr: 88.8% -- replaced with /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-mgr.smithi078.rrfxge.log.gz 2024-09-16T09:55:29.440 INFO:teuthology.orchestra.run.smithi078.stderr: 91.8% -- replaced with /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-mon.smithi078.log.gz 2024-09-16T09:55:39.686 INFO:teuthology.orchestra.run.smithi078.stderr: 94.5% -- replaced with /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-osd.2.log.gz 2024-09-16T09:55:41.437 INFO:teuthology.orchestra.run.smithi088.stderr: 93.9% -- replaced with /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-osd.4.log.gz 2024-09-16T09:55:42.588 INFO:teuthology.orchestra.run.smithi088.stderr: 94.2% -- replaced with /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-osd.6.log.gz 2024-09-16T09:55:44.457 INFO:teuthology.orchestra.run.smithi078.stderr: 94.1% -- replaced with /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-osd.0.log.gz 2024-09-16T09:55:45.185 INFO:teuthology.orchestra.run.smithi078.stderr: 94.2% -- replaced with /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-osd.1.log.gz 2024-09-16T09:55:45.378 INFO:teuthology.orchestra.run.smithi078.stderr: 94.0% -- replaced with /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-osd.3.log.gz 2024-09-16T09:55:45.380 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-16T09:55:45.380 INFO:teuthology.orchestra.run.smithi078.stderr:real 0m17.344s 2024-09-16T09:55:45.380 INFO:teuthology.orchestra.run.smithi078.stderr:user 1m1.790s 2024-09-16T09:55:45.380 INFO:teuthology.orchestra.run.smithi078.stderr:sys 0m2.475s 2024-09-16T09:55:47.298 INFO:teuthology.orchestra.run.smithi088.stderr: 94.2% -- replaced with /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-osd.5.log.gz 2024-09-16T09:55:50.986 INFO:teuthology.orchestra.run.smithi088.stderr: 94.1% -- replaced with /var/log/ceph/a70c765e-740d-11ef-bceb-c7b262605968/ceph-osd.7.log.gz 2024-09-16T09:55:50.988 INFO:teuthology.orchestra.run.smithi088.stderr: 2024-09-16T09:55:50.988 INFO:teuthology.orchestra.run.smithi088.stderr:real 0m22.951s 2024-09-16T09:55:50.988 INFO:teuthology.orchestra.run.smithi088.stderr:user 1m7.750s 2024-09-16T09:55:50.988 INFO:teuthology.orchestra.run.smithi088.stderr:sys 0m2.652s 2024-09-16T09:55:50.989 INFO:tasks.cephadm:Archiving logs... 2024-09-16T09:55:50.989 DEBUG:teuthology.misc:Transferring archived files from smithi078:/var/log/ceph to /home/teuthworker/archive/avan-2024-09-16_08:23:53-orch:cephadm-wip-athakkar-testing-2024-09-12-1857-distro-default-smithi/7907040/remote/smithi078/log 2024-09-16T09:55:50.990 DEBUG:teuthology.orchestra.run.smithi078:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-16T09:55:59.614 DEBUG:teuthology.misc:Transferring archived files from smithi088:/var/log/ceph to /home/teuthworker/archive/avan-2024-09-16_08:23:53-orch:cephadm-wip-athakkar-testing-2024-09-12-1857-distro-default-smithi/7907040/remote/smithi088/log 2024-09-16T09:55:59.615 DEBUG:teuthology.orchestra.run.smithi088:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-16T09:56:08.948 INFO:tasks.cephadm:Removing cluster... 2024-09-16T09:56:08.952 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid a70c765e-740d-11ef-bceb-c7b262605968 --force 2024-09-16T09:56:09.317 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid a70c765e-740d-11ef-bceb-c7b262605968 --force 2024-09-16T09:56:09.676 INFO:tasks.cephadm:Removing cephadm ... 2024-09-16T09:56:09.676 DEBUG:teuthology.orchestra.run.smithi078:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-16T09:56:09.692 DEBUG:teuthology.orchestra.run.smithi088:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-16T09:56:09.707 INFO:tasks.cephadm:Teardown complete 2024-09-16T09:56:09.708 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-16T09:56:09.720 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-16T09:56:09.721 DEBUG:teuthology.orchestra.run.smithi078:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-16T09:56:09.734 DEBUG:teuthology.orchestra.run.smithi088:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-16T09:56:09.746 INFO:teuthology.orchestra.run.smithi078.stderr:bash: line 1: ntpq: command not found 2024-09-16T09:56:09.751 INFO:teuthology.orchestra.run.smithi078.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-16T09:56:09.752 INFO:teuthology.orchestra.run.smithi078.stdout:=============================================================================== 2024-09-16T09:56:09.752 INFO:teuthology.orchestra.run.smithi078.stdout:^* hv01.front.sepia.ceph.com 3 6 377 8 -425us[ -355us] +/- 50ms 2024-09-16T09:56:09.752 INFO:teuthology.orchestra.run.smithi078.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 4 +1028us[+1028us] +/- 53ms 2024-09-16T09:56:09.752 INFO:teuthology.orchestra.run.smithi078.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 4 -575us[ -575us] +/- 75ms 2024-09-16T09:56:09.752 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-16T09:56:09.761 INFO:teuthology.orchestra.run.smithi088.stderr:bash: line 1: ntpq: command not found 2024-09-16T09:56:09.766 INFO:teuthology.orchestra.run.smithi088.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-16T09:56:09.766 INFO:teuthology.orchestra.run.smithi088.stdout:=============================================================================== 2024-09-16T09:56:09.766 INFO:teuthology.orchestra.run.smithi088.stdout:^* hv01.front.sepia.ceph.com 3 8 377 200 -358us[ -376us] +/- 47ms 2024-09-16T09:56:09.766 INFO:teuthology.orchestra.run.smithi088.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 7 +1104us[+1104us] +/- 53ms 2024-09-16T09:56:09.766 INFO:teuthology.orchestra.run.smithi088.stdout:^+ hv03.front.sepia.ceph.com 3 7 377 73 -478us[ -478us] +/- 74ms 2024-09-16T09:56:09.766 INFO:teuthology.orchestra.run.smithi088.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-16T09:56:09.767 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-16T09:56:09.818 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-16T09:56:09.819 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-16T09:56:09.900 DEBUG:teuthology.orchestra.run.smithi078:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-16T09:56:09.978 DEBUG:teuthology.orchestra.run.smithi088:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-16T09:56:10.050 DEBUG:teuthology.orchestra.run.smithi078:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-16T09:56:10.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-16T09:56:10.079 DEBUG:teuthology.orchestra.run.smithi088:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-16T09:56:10.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-16T09:56:10.107 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-16T09:56:10.116 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-16T09:56:10.200 INFO:teuthology.task.internal:Duration was 2049.090814 seconds 2024-09-16T09:56:10.201 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-16T09:56:10.275 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-16T09:56:10.276 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-16T09:56:10.279 DEBUG:teuthology.orchestra.run.smithi088:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-16T09:56:10.315 INFO:teuthology.orchestra.run.smithi078.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-16T09:56:10.317 INFO:teuthology.orchestra.run.smithi088.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-16T09:56:10.609 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-16T09:56:10.609 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi078.front.sepia.ceph.com 2024-09-16T09:56:10.610 DEBUG:teuthology.orchestra.run.smithi078:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-16T09:56:10.669 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi088.front.sepia.ceph.com 2024-09-16T09:56:10.669 DEBUG:teuthology.orchestra.run.smithi088:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-16T09:56:10.694 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-16T09:56:10.694 DEBUG:teuthology.orchestra.run.smithi078:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-16T09:56:10.711 DEBUG:teuthology.orchestra.run.smithi088:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-16T09:56:10.761 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-16T09:56:10.761 DEBUG:teuthology.orchestra.run.smithi078:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-16T09:56:10.777 DEBUG:teuthology.orchestra.run.smithi088:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-16T09:56:24.498 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-16T09:56:24.510 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-16T09:56:24.510 DEBUG:teuthology.orchestra.run.smithi078:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-16T09:56:24.535 DEBUG:teuthology.orchestra.run.smithi088:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-16T09:56:24.566 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-16T09:56:24.584 DEBUG:teuthology.orchestra.run.smithi078:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-16T09:56:24.587 DEBUG:teuthology.orchestra.run.smithi088:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-16T09:56:24.607 INFO:teuthology.orchestra.run.smithi078.stdout:kernel.core_pattern = core 2024-09-16T09:56:24.634 INFO:teuthology.orchestra.run.smithi088.stdout:kernel.core_pattern = core 2024-09-16T09:56:24.659 DEBUG:teuthology.orchestra.run.smithi078:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-16T09:56:24.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-16T09:56:24.685 DEBUG:teuthology.orchestra.run.smithi088:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-16T09:56:24.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-16T09:56:24.716 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-16T09:56:24.726 INFO:teuthology.task.internal:Transferring archived files... 2024-09-16T09:56:24.727 DEBUG:teuthology.misc:Transferring archived files from smithi078:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/avan-2024-09-16_08:23:53-orch:cephadm-wip-athakkar-testing-2024-09-12-1857-distro-default-smithi/7907040/remote/smithi078 2024-09-16T09:56:24.727 DEBUG:teuthology.orchestra.run.smithi078:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-16T09:56:24.787 DEBUG:teuthology.misc:Transferring archived files from smithi088:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/avan-2024-09-16_08:23:53-orch:cephadm-wip-athakkar-testing-2024-09-12-1857-distro-default-smithi/7907040/remote/smithi088 2024-09-16T09:56:24.787 DEBUG:teuthology.orchestra.run.smithi088:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-16T09:56:24.831 INFO:teuthology.task.internal:Removing archive directory... 2024-09-16T09:56:24.832 DEBUG:teuthology.orchestra.run.smithi078:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-16T09:56:24.834 DEBUG:teuthology.orchestra.run.smithi088:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-16T09:56:24.882 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-16T09:56:24.894 INFO:teuthology.task.internal:Not uploading archives. 2024-09-16T09:56:24.894 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-16T09:56:24.918 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-16T09:56:24.919 DEBUG:teuthology.orchestra.run.smithi078:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-16T09:56:24.921 DEBUG:teuthology.orchestra.run.smithi088:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-16T09:56:24.933 INFO:teuthology.orchestra.run.smithi078.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 16 09:56 /home/ubuntu/cephtest 2024-09-16T09:56:24.939 INFO:teuthology.orchestra.run.smithi088.stdout: 265324 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 16 09:56 /home/ubuntu/cephtest 2024-09-16T09:56:24.940 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-16T09:56:24.955 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-16T09:56:25.023 INFO:teuthology.run:Summary data: description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 2049.09081363678 owner: scheduled_avan@teuthology success: true 2024-09-16T09:56:25.024 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-16T09:56:25.106 INFO:teuthology.run:pass