2024-09-21T12:09:44.073 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-21T12:09:44.090 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-21T12:09:44.166 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/gabrioux-2024-09-21_11:23:40-orch:cephadm-wip-guits-main-2024-09-21-0842-distro-default-smithi/7913924 branch: wip-guits-main-2024-09-21-0842 description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: gabrioux@ibm.com first_in_suite: false job_id: '7913924' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: gabrioux-2024-09-21_11:23:40-orch:cephadm-wip-guits-main-2024-09-21-0842-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-guits-main-2024-09-21-0842 ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - CEPHADM_REFRESH_FAILED log-only-match: - CEPHADM_ sha1: 441784946648ff53dd310293f9dd752afeeea72a ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 441784946648ff53dd310293f9dd752afeeea72a selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: wip-guits-main-2024-09-21-0842 sha1: 441784946648ff53dd310293f9dd752afeeea72a owner: scheduled_gabrioux@teuthology priority: 70 repo: https://git.ceph.com/ceph-ci.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 5896 sha1: 441784946648ff53dd310293f9dd752afeeea72a sleep_before_teardown: 0 subset: 1/8 suite: orch:cephadm suite_branch: wip-guits-main-2024-09-21-0842 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_441784946648ff53dd310293f9dd752afeeea72a/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 441784946648ff53dd310293f9dd752afeeea72a targets: smithi078.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDH19bgTkK15EQ/gc7FW4+aXgMoT/CFCQlfOwkcX9iiXdyPmsabYsPwzOkbngQ9N+ke3lRpHMi3Qdk695b8Yer0= smithi148.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLZj3DhJqmfT/nYgh3AmqFJfAEDWBI2Y5yS+YLz7nBmZQ6y56oBVzZebyB1wxJTZFeLQfzh0qlX5ckrN8Y1nIdM= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-21_11:23:40 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: gabrioux verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2902 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-09-21T12:09:44.166 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_441784946648ff53dd310293f9dd752afeeea72a/qa; will attempt to use it 2024-09-21T12:09:44.167 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_441784946648ff53dd310293f9dd752afeeea72a/qa/tasks 2024-09-21T12:09:44.167 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-21T12:09:44.169 INFO:teuthology.task.internal:Checking packages... 2024-09-21T12:09:44.189 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '441784946648ff53dd310293f9dd752afeeea72a' 2024-09-21T12:09:44.190 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T12:09:44.190 INFO:teuthology.packaging:ref: None 2024-09-21T12:09:44.190 INFO:teuthology.packaging:tag: None 2024-09-21T12:09:44.190 INFO:teuthology.packaging:branch: wip-guits-main-2024-09-21-0842 2024-09-21T12:09:44.190 INFO:teuthology.packaging:sha1: 441784946648ff53dd310293f9dd752afeeea72a 2024-09-21T12:09:44.190 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-guits-main-2024-09-21-0842 2024-09-21T12:09:44.331 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-5104.g44178494 2024-09-21T12:09:44.333 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-21T12:09:44.339 INFO:teuthology.task.internal:no buildpackages task found 2024-09-21T12:09:44.339 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-21T12:09:44.383 INFO:teuthology.task.internal:Saving configuration 2024-09-21T12:09:44.397 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-21T12:09:44.404 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-21T12:09:44.427 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi078.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-09-21_11:23:40-orch:cephadm-wip-guits-main-2024-09-21-0842-distro-default-smithi/7913924', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-21 12:04:44.797227', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDH19bgTkK15EQ/gc7FW4+aXgMoT/CFCQlfOwkcX9iiXdyPmsabYsPwzOkbngQ9N+ke3lRpHMi3Qdk695b8Yer0='} 2024-09-21T12:09:44.447 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi148.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-09-21_11:23:40-orch:cephadm-wip-guits-main-2024-09-21-0842-distro-default-smithi/7913924', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-21 12:04:44.798495', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLZj3DhJqmfT/nYgh3AmqFJfAEDWBI2Y5yS+YLz7nBmZQ6y56oBVzZebyB1wxJTZFeLQfzh0qlX5ckrN8Y1nIdM='} 2024-09-21T12:09:44.447 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-21T12:09:44.454 INFO:teuthology.task.internal:roles: ubuntu@smithi078.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-09-21T12:09:44.454 INFO:teuthology.task.internal:roles: ubuntu@smithi148.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-09-21T12:09:44.454 INFO:teuthology.run_tasks:Running task console_log... 2024-09-21T12:09:44.514 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fdbba3f8280>, signals=[15]) 2024-09-21T12:09:44.514 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-21T12:09:44.520 INFO:teuthology.task.internal:Opening connections... 2024-09-21T12:09:44.520 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi078.front.sepia.ceph.com 2024-09-21T12:09:44.521 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T12:09:44.612 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi148.front.sepia.ceph.com 2024-09-21T12:09:44.613 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi148.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T12:09:44.686 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-21T12:09:44.692 DEBUG:teuthology.orchestra.run.smithi078:> uname -m 2024-09-21T12:09:44.710 INFO:teuthology.orchestra.run.smithi078.stdout:x86_64 2024-09-21T12:09:44.711 DEBUG:teuthology.orchestra.run.smithi078:> cat /etc/os-release 2024-09-21T12:09:44.769 INFO:teuthology.orchestra.run.smithi078.stdout:NAME="CentOS Stream" 2024-09-21T12:09:44.769 INFO:teuthology.orchestra.run.smithi078.stdout:VERSION="9" 2024-09-21T12:09:44.769 INFO:teuthology.orchestra.run.smithi078.stdout:ID="centos" 2024-09-21T12:09:44.769 INFO:teuthology.orchestra.run.smithi078.stdout:ID_LIKE="rhel fedora" 2024-09-21T12:09:44.769 INFO:teuthology.orchestra.run.smithi078.stdout:VERSION_ID="9" 2024-09-21T12:09:44.769 INFO:teuthology.orchestra.run.smithi078.stdout:PLATFORM_ID="platform:el9" 2024-09-21T12:09:44.770 INFO:teuthology.orchestra.run.smithi078.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-21T12:09:44.770 INFO:teuthology.orchestra.run.smithi078.stdout:ANSI_COLOR="0;31" 2024-09-21T12:09:44.770 INFO:teuthology.orchestra.run.smithi078.stdout:LOGO="fedora-logo-icon" 2024-09-21T12:09:44.770 INFO:teuthology.orchestra.run.smithi078.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-21T12:09:44.770 INFO:teuthology.orchestra.run.smithi078.stdout:HOME_URL="https://centos.org/" 2024-09-21T12:09:44.770 INFO:teuthology.orchestra.run.smithi078.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-21T12:09:44.770 INFO:teuthology.orchestra.run.smithi078.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-21T12:09:44.770 INFO:teuthology.orchestra.run.smithi078.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-21T12:09:44.771 INFO:teuthology.lock.ops:Updating smithi078.front.sepia.ceph.com on lock server 2024-09-21T12:09:44.793 DEBUG:teuthology.orchestra.run.smithi148:> uname -m 2024-09-21T12:09:44.812 INFO:teuthology.orchestra.run.smithi148.stdout:x86_64 2024-09-21T12:09:44.812 DEBUG:teuthology.orchestra.run.smithi148:> cat /etc/os-release 2024-09-21T12:09:44.870 INFO:teuthology.orchestra.run.smithi148.stdout:NAME="CentOS Stream" 2024-09-21T12:09:44.871 INFO:teuthology.orchestra.run.smithi148.stdout:VERSION="9" 2024-09-21T12:09:44.871 INFO:teuthology.orchestra.run.smithi148.stdout:ID="centos" 2024-09-21T12:09:44.871 INFO:teuthology.orchestra.run.smithi148.stdout:ID_LIKE="rhel fedora" 2024-09-21T12:09:44.871 INFO:teuthology.orchestra.run.smithi148.stdout:VERSION_ID="9" 2024-09-21T12:09:44.871 INFO:teuthology.orchestra.run.smithi148.stdout:PLATFORM_ID="platform:el9" 2024-09-21T12:09:44.871 INFO:teuthology.orchestra.run.smithi148.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-21T12:09:44.871 INFO:teuthology.orchestra.run.smithi148.stdout:ANSI_COLOR="0;31" 2024-09-21T12:09:44.871 INFO:teuthology.orchestra.run.smithi148.stdout:LOGO="fedora-logo-icon" 2024-09-21T12:09:44.871 INFO:teuthology.orchestra.run.smithi148.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-21T12:09:44.871 INFO:teuthology.orchestra.run.smithi148.stdout:HOME_URL="https://centos.org/" 2024-09-21T12:09:44.871 INFO:teuthology.orchestra.run.smithi148.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-21T12:09:44.871 INFO:teuthology.orchestra.run.smithi148.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-21T12:09:44.872 INFO:teuthology.orchestra.run.smithi148.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-21T12:09:44.872 INFO:teuthology.lock.ops:Updating smithi148.front.sepia.ceph.com on lock server 2024-09-21T12:09:44.893 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-21T12:09:44.900 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-21T12:09:44.906 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-21T12:09:44.906 DEBUG:teuthology.orchestra.run.smithi078:> test '!' -e /home/ubuntu/cephtest 2024-09-21T12:09:44.909 DEBUG:teuthology.orchestra.run.smithi148:> test '!' -e /home/ubuntu/cephtest 2024-09-21T12:09:44.926 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-21T12:09:44.932 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-21T12:09:44.932 DEBUG:teuthology.orchestra.run.smithi078:> test -z $(ls -A /var/lib/ceph) 2024-09-21T12:09:44.967 DEBUG:teuthology.orchestra.run.smithi148:> test -z $(ls -A /var/lib/ceph) 2024-09-21T12:09:45.058 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-21T12:09:45.108 INFO:teuthology.run_tasks:Running task kernel... 2024-09-21T12:09:45.122 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-21T12:09:45.122 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-21T12:09:45.122 DEBUG:teuthology.orchestra.run.smithi078:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-21T12:09:45.123 DEBUG:teuthology.orchestra.run.smithi148:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-21T12:09:45.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T12:09:45.140 DEBUG:teuthology.orchestra.run.smithi078:> uname -r 2024-09-21T12:09:45.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T12:09:45.141 DEBUG:teuthology.orchestra.run.smithi148:> uname -r 2024-09-21T12:09:45.197 INFO:teuthology.orchestra.run.smithi078.stdout:5.14.0-503.el9.x86_64 2024-09-21T12:09:45.197 INFO:teuthology.task.kernel:Running kernel on smithi078: 5.14.0-503.el9.x86_64 2024-09-21T12:09:45.197 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum install -y kernel 2024-09-21T12:09:45.198 INFO:teuthology.orchestra.run.smithi148.stdout:5.14.0-503.el9.x86_64 2024-09-21T12:09:45.199 INFO:teuthology.task.kernel:Running kernel on smithi148: 5.14.0-503.el9.x86_64 2024-09-21T12:09:45.199 DEBUG:teuthology.orchestra.run.smithi148:> sudo yum install -y kernel 2024-09-21T12:09:48.776 INFO:teuthology.orchestra.run.smithi148.stdout:CentOS Stream 9 - BaseOS 4.7 MB/s | 8.3 MB 00:01 2024-09-21T12:09:48.820 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - BaseOS 4.3 MB/s | 8.3 MB 00:01 2024-09-21T12:09:50.644 INFO:teuthology.orchestra.run.smithi148.stdout:CentOS Stream 9 - AppStream 26 MB/s | 20 MB 00:00 2024-09-21T12:09:55.086 INFO:teuthology.orchestra.run.smithi148.stdout:CentOS Stream 9 - CRB 10 MB/s | 6.5 MB 00:00 2024-09-21T12:09:55.087 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - AppStream 3.9 MB/s | 20 MB 00:05 2024-09-21T12:09:56.695 INFO:teuthology.orchestra.run.smithi148.stdout:CentOS Stream 9 - Extras packages 48 kB/s | 19 kB 00:00 2024-09-21T12:09:57.891 INFO:teuthology.orchestra.run.smithi148.stdout:Extra Packages for Enterprise Linux 21 MB/s | 23 MB 00:01 2024-09-21T12:09:59.471 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - CRB 12 MB/s | 6.5 MB 00:00 2024-09-21T12:10:00.980 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - Extras packages 66 kB/s | 19 kB 00:00 2024-09-21T12:10:02.001 INFO:teuthology.orchestra.run.smithi078.stdout:Extra Packages for Enterprise Linux 25 MB/s | 23 MB 00:00 2024-09-21T12:10:03.272 INFO:teuthology.orchestra.run.smithi148.stdout:lab-extras 13 kB/s | 1.7 kB 00:00 2024-09-21T12:10:05.203 INFO:teuthology.orchestra.run.smithi148.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-21T12:10:05.203 INFO:teuthology.orchestra.run.smithi148.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-21T12:10:05.299 INFO:teuthology.orchestra.run.smithi148.stdout:Dependencies resolved. 2024-09-21T12:10:05.308 INFO:teuthology.orchestra.run.smithi148.stdout:================================================================================ 2024-09-21T12:10:05.308 INFO:teuthology.orchestra.run.smithi148.stdout: Package Architecture Version Repository Size 2024-09-21T12:10:05.308 INFO:teuthology.orchestra.run.smithi148.stdout:================================================================================ 2024-09-21T12:10:05.309 INFO:teuthology.orchestra.run.smithi148.stdout:Installing: 2024-09-21T12:10:05.309 INFO:teuthology.orchestra.run.smithi148.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-21T12:10:05.309 INFO:teuthology.orchestra.run.smithi148.stdout:Installing dependencies: 2024-09-21T12:10:05.309 INFO:teuthology.orchestra.run.smithi148.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-21T12:10:05.309 INFO:teuthology.orchestra.run.smithi148.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-21T12:10:05.309 INFO:teuthology.orchestra.run.smithi148.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-21T12:10:05.309 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-09-21T12:10:05.309 INFO:teuthology.orchestra.run.smithi148.stdout:Transaction Summary 2024-09-21T12:10:05.309 INFO:teuthology.orchestra.run.smithi148.stdout:================================================================================ 2024-09-21T12:10:05.309 INFO:teuthology.orchestra.run.smithi148.stdout:Install 4 Packages 2024-09-21T12:10:05.309 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-09-21T12:10:05.310 INFO:teuthology.orchestra.run.smithi148.stdout:Total download size: 79 M 2024-09-21T12:10:05.310 INFO:teuthology.orchestra.run.smithi148.stdout:Installed size: 126 M 2024-09-21T12:10:05.310 INFO:teuthology.orchestra.run.smithi148.stdout:Downloading Packages: 2024-09-21T12:10:05.781 INFO:teuthology.orchestra.run.smithi148.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 240 kB/s | 36 kB 00:00 2024-09-21T12:10:07.415 INFO:teuthology.orchestra.run.smithi078.stdout:lab-extras 21 kB/s | 1.7 kB 00:00 2024-09-21T12:10:07.540 INFO:teuthology.orchestra.run.smithi148.stdout:(2/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 18 MB/s | 35 MB 00:01 2024-09-21T12:10:08.132 INFO:teuthology.orchestra.run.smithi148.stdout:(3/4): kernel-modules-core-5.14.0-508.el9.x86_6 12 MB/s | 29 MB 00:02 2024-09-21T12:10:09.107 INFO:teuthology.orchestra.run.smithi148.stdout:(4/4): kernel-core-5.14.0-508.el9.x86_64.rpm 4.5 MB/s | 16 MB 00:03 2024-09-21T12:10:09.107 INFO:teuthology.orchestra.run.smithi148.stdout:-------------------------------------------------------------------------------- 2024-09-21T12:10:09.107 INFO:teuthology.orchestra.run.smithi148.stdout:Total 21 MB/s | 79 MB 00:03 2024-09-21T12:10:09.322 INFO:teuthology.orchestra.run.smithi078.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-21T12:10:09.322 INFO:teuthology.orchestra.run.smithi078.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-21T12:10:09.399 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-09-21T12:10:09.410 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T12:10:09.410 INFO:teuthology.orchestra.run.smithi078.stdout: Package Architecture Version Repository Size 2024-09-21T12:10:09.410 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T12:10:09.410 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-09-21T12:10:09.410 INFO:teuthology.orchestra.run.smithi078.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-21T12:10:09.410 INFO:teuthology.orchestra.run.smithi078.stdout:Installing dependencies: 2024-09-21T12:10:09.411 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-21T12:10:09.411 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-21T12:10:09.411 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-21T12:10:09.411 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T12:10:09.411 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-09-21T12:10:09.411 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-09-21T12:10:09.411 INFO:teuthology.orchestra.run.smithi078.stdout:Install 4 Packages 2024-09-21T12:10:09.411 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T12:10:09.412 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 79 M 2024-09-21T12:10:09.412 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 126 M 2024-09-21T12:10:09.412 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-09-21T12:10:09.783 INFO:teuthology.orchestra.run.smithi078.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 380 kB/s | 36 kB 00:00 2024-09-21T12:10:09.792 INFO:teuthology.orchestra.run.smithi148.stdout:Running transaction check 2024-09-21T12:10:10.146 INFO:teuthology.orchestra.run.smithi148.stdout:Transaction check succeeded. 2024-09-21T12:10:10.146 INFO:teuthology.orchestra.run.smithi148.stdout:Running transaction test 2024-09-21T12:10:10.184 INFO:teuthology.orchestra.run.smithi078.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 32 MB/s | 16 MB 00:00 2024-09-21T12:10:10.675 INFO:teuthology.orchestra.run.smithi078.stdout:(3/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 35 MB/s | 35 MB 00:00 2024-09-21T12:10:10.934 INFO:teuthology.orchestra.run.smithi078.stdout:(4/4): kernel-modules-core-5.14.0-508.el9.x86_6 25 MB/s | 29 MB 00:01 2024-09-21T12:10:10.934 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-09-21T12:10:10.935 INFO:teuthology.orchestra.run.smithi078.stdout:Total 52 MB/s | 79 MB 00:01 2024-09-21T12:10:11.136 INFO:teuthology.orchestra.run.smithi148.stdout:Transaction test succeeded. 2024-09-21T12:10:11.136 INFO:teuthology.orchestra.run.smithi148.stdout:Running transaction 2024-09-21T12:10:11.618 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-09-21T12:10:11.967 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-09-21T12:10:11.967 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-09-21T12:10:12.944 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-09-21T12:10:12.945 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-09-21T12:10:13.605 INFO:teuthology.orchestra.run.smithi148.stdout: Preparing : 1/1 2024-09-21T12:10:14.350 INFO:teuthology.orchestra.run.smithi148.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-21T12:10:14.432 INFO:teuthology.orchestra.run.smithi148.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T12:10:15.347 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-09-21T12:10:16.036 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-21T12:10:16.127 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T12:10:17.186 INFO:teuthology.orchestra.run.smithi148.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T12:10:17.331 INFO:teuthology.orchestra.run.smithi148.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T12:10:18.911 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T12:10:19.039 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T12:10:22.175 INFO:teuthology.orchestra.run.smithi148.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T12:10:22.292 INFO:teuthology.orchestra.run.smithi148.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T12:10:23.871 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T12:10:23.997 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T12:10:27.122 INFO:teuthology.orchestra.run.smithi148.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T12:10:28.851 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T12:11:05.398 INFO:teuthology.orchestra.run.smithi148.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T12:11:05.438 INFO:teuthology.orchestra.run.smithi148.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-21T12:11:05.737 INFO:teuthology.orchestra.run.smithi148.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T12:11:05.737 INFO:teuthology.orchestra.run.smithi148.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-21T12:11:05.737 INFO:teuthology.orchestra.run.smithi148.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T12:11:05.737 INFO:teuthology.orchestra.run.smithi148.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T12:11:06.563 INFO:teuthology.orchestra.run.smithi148.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T12:11:06.563 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-09-21T12:11:06.563 INFO:teuthology.orchestra.run.smithi148.stdout:Installed: 2024-09-21T12:11:06.563 INFO:teuthology.orchestra.run.smithi148.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-21T12:11:06.563 INFO:teuthology.orchestra.run.smithi148.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-21T12:11:06.563 INFO:teuthology.orchestra.run.smithi148.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-21T12:11:06.564 INFO:teuthology.orchestra.run.smithi148.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-21T12:11:06.564 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-09-21T12:11:06.564 INFO:teuthology.orchestra.run.smithi148.stdout:Complete! 2024-09-21T12:11:06.930 DEBUG:teuthology.orchestra.run.smithi148:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T12:11:07.011 INFO:teuthology.orchestra.run.smithi148.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T12:11:07.011 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T12:11:07.011 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-21T12:11:07.012 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-21T12:11:07.012 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-21T12:11:07.012 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-09-21T12:11:07.012 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi148.front.sepia.ceph.com, path=None, version=distro) 2024-09-21T12:11:07.012 DEBUG:teuthology.orchestra.run.smithi148:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T12:11:07.071 INFO:teuthology.orchestra.run.smithi148.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T12:11:07.071 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T12:11:07.071 DEBUG:teuthology.orchestra.run.smithi148:> sudo rpm -qi grub2-tools 2024-09-21T12:11:07.103 INFO:teuthology.orchestra.run.smithi148.stdout:Name : grub2-tools 2024-09-21T12:11:07.103 INFO:teuthology.orchestra.run.smithi148.stdout:Epoch : 1 2024-09-21T12:11:07.103 INFO:teuthology.orchestra.run.smithi148.stdout:Version : 2.06 2024-09-21T12:11:07.103 INFO:teuthology.orchestra.run.smithi148.stdout:Release : 82.el9 2024-09-21T12:11:07.103 INFO:teuthology.orchestra.run.smithi148.stdout:Architecture: x86_64 2024-09-21T12:11:07.103 INFO:teuthology.orchestra.run.smithi148.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-21T12:11:07.103 INFO:teuthology.orchestra.run.smithi148.stdout:Group : Unspecified 2024-09-21T12:11:07.103 INFO:teuthology.orchestra.run.smithi148.stdout:Size : 8274670 2024-09-21T12:11:07.103 INFO:teuthology.orchestra.run.smithi148.stdout:License : GPLv3+ 2024-09-21T12:11:07.104 INFO:teuthology.orchestra.run.smithi148.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-21T12:11:07.104 INFO:teuthology.orchestra.run.smithi148.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-21T12:11:07.104 INFO:teuthology.orchestra.run.smithi148.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-21T12:11:07.104 INFO:teuthology.orchestra.run.smithi148.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-21T12:11:07.104 INFO:teuthology.orchestra.run.smithi148.stdout:Packager : builder@centos.org 2024-09-21T12:11:07.104 INFO:teuthology.orchestra.run.smithi148.stdout:Vendor : CentOS 2024-09-21T12:11:07.104 INFO:teuthology.orchestra.run.smithi148.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-21T12:11:07.104 INFO:teuthology.orchestra.run.smithi148.stdout:Summary : Support tools for GRUB. 2024-09-21T12:11:07.104 INFO:teuthology.orchestra.run.smithi148.stdout:Description : 2024-09-21T12:11:07.104 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-09-21T12:11:07.105 INFO:teuthology.orchestra.run.smithi148.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-21T12:11:07.105 INFO:teuthology.orchestra.run.smithi148.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-21T12:11:07.105 INFO:teuthology.orchestra.run.smithi148.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-21T12:11:07.105 INFO:teuthology.orchestra.run.smithi148.stdout:hardware devices. 2024-09-21T12:11:07.105 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-09-21T12:11:07.105 INFO:teuthology.orchestra.run.smithi148.stdout:This subpackage provides tools for support of all platforms. 2024-09-21T12:11:07.106 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-21T12:11:07.106 INFO:teuthology.task.kernel:Updating grub on smithi148 to boot 5.14.0-508.el9.x86_64 2024-09-21T12:11:07.106 DEBUG:teuthology.orchestra.run.smithi148:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-21T12:11:07.135 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T12:11:07.175 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-21T12:11:07.541 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T12:11:07.541 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-21T12:11:07.542 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T12:11:07.542 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T12:11:07.806 INFO:teuthology.orchestra.run.smithi148.stderr:Generating grub configuration file ... 2024-09-21T12:11:08.388 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T12:11:08.388 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T12:11:08.388 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-09-21T12:11:08.388 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-21T12:11:08.388 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-21T12:11:08.388 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-21T12:11:08.388 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-21T12:11:08.389 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T12:11:08.389 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-09-21T12:11:08.741 DEBUG:teuthology.orchestra.run.smithi078:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T12:11:08.819 INFO:teuthology.orchestra.run.smithi078.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T12:11:08.819 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T12:11:08.820 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-21T12:11:08.820 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-21T12:11:08.820 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-21T12:11:08.820 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-09-21T12:11:08.820 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi078.front.sepia.ceph.com, path=None, version=distro) 2024-09-21T12:11:08.820 DEBUG:teuthology.orchestra.run.smithi078:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T12:11:08.878 INFO:teuthology.orchestra.run.smithi078.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T12:11:08.879 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T12:11:08.879 DEBUG:teuthology.orchestra.run.smithi078:> sudo rpm -qi grub2-tools 2024-09-21T12:11:08.917 INFO:teuthology.orchestra.run.smithi078.stdout:Name : grub2-tools 2024-09-21T12:11:08.917 INFO:teuthology.orchestra.run.smithi078.stdout:Epoch : 1 2024-09-21T12:11:08.917 INFO:teuthology.orchestra.run.smithi078.stdout:Version : 2.06 2024-09-21T12:11:08.917 INFO:teuthology.orchestra.run.smithi078.stdout:Release : 82.el9 2024-09-21T12:11:08.917 INFO:teuthology.orchestra.run.smithi078.stdout:Architecture: x86_64 2024-09-21T12:11:08.917 INFO:teuthology.orchestra.run.smithi078.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-21T12:11:08.917 INFO:teuthology.orchestra.run.smithi078.stdout:Group : Unspecified 2024-09-21T12:11:08.917 INFO:teuthology.orchestra.run.smithi078.stdout:Size : 8274670 2024-09-21T12:11:08.917 INFO:teuthology.orchestra.run.smithi078.stdout:License : GPLv3+ 2024-09-21T12:11:08.918 INFO:teuthology.orchestra.run.smithi078.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-21T12:11:08.918 INFO:teuthology.orchestra.run.smithi078.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-21T12:11:08.918 INFO:teuthology.orchestra.run.smithi078.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-21T12:11:08.918 INFO:teuthology.orchestra.run.smithi078.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-21T12:11:08.918 INFO:teuthology.orchestra.run.smithi078.stdout:Packager : builder@centos.org 2024-09-21T12:11:08.918 INFO:teuthology.orchestra.run.smithi078.stdout:Vendor : CentOS 2024-09-21T12:11:08.918 INFO:teuthology.orchestra.run.smithi078.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-21T12:11:08.918 INFO:teuthology.orchestra.run.smithi078.stdout:Summary : Support tools for GRUB. 2024-09-21T12:11:08.918 INFO:teuthology.orchestra.run.smithi078.stdout:Description : 2024-09-21T12:11:08.918 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T12:11:08.918 INFO:teuthology.orchestra.run.smithi078.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-21T12:11:08.918 INFO:teuthology.orchestra.run.smithi078.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-21T12:11:08.918 INFO:teuthology.orchestra.run.smithi078.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-21T12:11:08.918 INFO:teuthology.orchestra.run.smithi078.stdout:hardware devices. 2024-09-21T12:11:08.919 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T12:11:08.919 INFO:teuthology.orchestra.run.smithi078.stdout:This subpackage provides tools for support of all platforms. 2024-09-21T12:11:08.919 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-21T12:11:08.919 INFO:teuthology.task.kernel:Updating grub on smithi078 to boot 5.14.0-508.el9.x86_64 2024-09-21T12:11:08.920 DEBUG:teuthology.orchestra.run.smithi078:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-21T12:11:09.125 INFO:teuthology.orchestra.run.smithi148.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-21T12:11:09.157 INFO:teuthology.orchestra.run.smithi148.stderr:done 2024-09-21T12:11:09.159 DEBUG:teuthology.orchestra.run.smithi148:> mktemp 2024-09-21T12:11:09.175 INFO:teuthology.orchestra.run.smithi148.stdout:/tmp/tmp.IIgM1Xf7ms 2024-09-21T12:11:09.175 DEBUG:teuthology.orchestra.run.smithi148:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.IIgM1Xf7ms 2024-09-21T12:11:09.242 DEBUG:teuthology.orchestra.run.smithi148:> sudo chmod 0666 /tmp/tmp.IIgM1Xf7ms 2024-09-21T12:11:09.385 DEBUG:teuthology.orchestra.remote:smithi148:/tmp/tmp.IIgM1Xf7ms is 6KB 2024-09-21T12:11:09.433 DEBUG:teuthology.orchestra.run.smithi148:> rm -fr /tmp/tmp.IIgM1Xf7ms 2024-09-21T12:11:09.449 DEBUG:teuthology.orchestra.run.smithi148:> sudo /bin/ls /boot/loader/entries || true 2024-09-21T12:11:09.511 INFO:teuthology.orchestra.run.smithi148.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-21T12:11:09.511 INFO:teuthology.orchestra.run.smithi148.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-21T12:11:09.512 INFO:teuthology.orchestra.run.smithi148.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-21T12:11:09.512 INFO:teuthology.orchestra.run.smithi148.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-21T12:11:09.513 DEBUG:teuthology.orchestra.run.smithi148:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-21T12:11:09.572 INFO:teuthology.orchestra.run.smithi078.stderr:Generating grub configuration file ... 2024-09-21T12:11:09.660 DEBUG:teuthology.orchestra.run.smithi148:> sudo shutdown -r now 2024-09-21T12:11:10.908 INFO:teuthology.orchestra.run.smithi078.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-21T12:11:10.940 INFO:teuthology.orchestra.run.smithi078.stderr:done 2024-09-21T12:11:10.952 DEBUG:teuthology.orchestra.run.smithi078:> mktemp 2024-09-21T12:11:10.985 INFO:teuthology.orchestra.run.smithi078.stdout:/tmp/tmp.3WSEWzE1Mh 2024-09-21T12:11:10.986 DEBUG:teuthology.orchestra.run.smithi078:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.3WSEWzE1Mh 2024-09-21T12:11:11.062 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 0666 /tmp/tmp.3WSEWzE1Mh 2024-09-21T12:11:11.210 DEBUG:teuthology.orchestra.remote:smithi078:/tmp/tmp.3WSEWzE1Mh is 6KB 2024-09-21T12:11:11.259 DEBUG:teuthology.orchestra.run.smithi078:> rm -fr /tmp/tmp.3WSEWzE1Mh 2024-09-21T12:11:11.274 DEBUG:teuthology.orchestra.run.smithi078:> sudo /bin/ls /boot/loader/entries || true 2024-09-21T12:11:11.336 INFO:teuthology.orchestra.run.smithi078.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-21T12:11:11.337 INFO:teuthology.orchestra.run.smithi078.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-21T12:11:11.337 INFO:teuthology.orchestra.run.smithi078.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-21T12:11:11.337 INFO:teuthology.orchestra.run.smithi078.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-21T12:11:11.338 DEBUG:teuthology.orchestra.run.smithi078:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-21T12:11:11.481 DEBUG:teuthology.orchestra.run.smithi078:> sudo shutdown -r now 2024-09-21T12:11:39.690 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-09-21T12:11:39.691 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi148.front.sepia.ceph.com' 2024-09-21T12:11:39.691 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi148.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T12:11:41.486 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-09-21T12:11:41.486 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-21T12:11:41.486 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T12:11:58.063 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.148 2024-09-21T12:11:59.855 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.78 2024-09-21T12:12:07.070 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi148.front.sepia.ceph.com' 2024-09-21T12:12:07.071 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi148.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T12:12:08.856 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-21T12:12:08.857 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T12:12:10.127 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.148 2024-09-21T12:12:11.919 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.78 2024-09-21T12:12:22.137 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi148.front.sepia.ceph.com' 2024-09-21T12:12:22.138 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi148.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T12:12:23.921 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-21T12:12:23.922 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T12:12:26.991 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.78 2024-09-21T12:12:40.559 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.148 2024-09-21T12:12:41.992 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-21T12:12:41.993 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T12:12:45.071 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.78 2024-09-21T12:12:55.566 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi148.front.sepia.ceph.com' 2024-09-21T12:12:55.567 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi148.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T12:12:56.024 DEBUG:teuthology.orchestra.run.smithi148:> true 2024-09-21T12:12:56.379 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi148.front.sepia.ceph.com' 2024-09-21T12:12:56.379 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-508.el9.x86_64"... 2024-09-21T12:12:56.379 DEBUG:teuthology.orchestra.run.smithi148:> uname -r 2024-09-21T12:12:56.394 INFO:teuthology.orchestra.run.smithi148.stdout:5.14.0-508.el9.x86_64 2024-09-21T12:12:56.394 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-21T12:12:56.395 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-21T12:12:56.395 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-21T12:12:57.395 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-09-21T12:12:57.395 DEBUG:teuthology.orchestra.run.smithi148:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-21T12:12:57.465 INFO:teuthology.orchestra.run.smithi148.stdout:ttyS1 2024-09-21T12:12:57.484 DEBUG:teuthology.parallel:result is None 2024-09-21T12:13:03.071 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-21T12:13:03.072 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T12:13:03.331 DEBUG:teuthology.orchestra.run.smithi078:> true 2024-09-21T12:13:03.677 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-21T12:13:03.677 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-508.el9.x86_64"... 2024-09-21T12:13:03.677 DEBUG:teuthology.orchestra.run.smithi078:> uname -r 2024-09-21T12:13:03.695 INFO:teuthology.orchestra.run.smithi078.stdout:5.14.0-508.el9.x86_64 2024-09-21T12:13:03.695 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-21T12:13:03.695 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-21T12:13:03.695 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-21T12:13:04.696 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-09-21T12:13:04.696 DEBUG:teuthology.orchestra.run.smithi078:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-21T12:13:04.761 INFO:teuthology.orchestra.run.smithi078.stdout:ttyS1 2024-09-21T12:13:04.784 DEBUG:teuthology.parallel:result is None 2024-09-21T12:13:04.785 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-21T12:13:04.793 INFO:teuthology.task.internal:Creating test directory... 2024-09-21T12:13:04.793 DEBUG:teuthology.orchestra.run.smithi078:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-21T12:13:04.795 DEBUG:teuthology.orchestra.run.smithi148:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-21T12:13:04.814 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-21T12:13:04.820 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-21T12:13:04.865 INFO:teuthology.task.internal:Creating archive directory... 2024-09-21T12:13:04.865 DEBUG:teuthology.orchestra.run.smithi078:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-21T12:13:04.868 DEBUG:teuthology.orchestra.run.smithi148:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-21T12:13:04.985 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-21T12:13:04.995 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-21T12:13:04.995 DEBUG:teuthology.orchestra.run.smithi078:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-21T12:13:04.998 DEBUG:teuthology.orchestra.run.smithi148:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-21T12:13:05.030 INFO:teuthology.orchestra.run.smithi148.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T12:13:05.033 INFO:teuthology.orchestra.run.smithi078.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T12:13:05.041 INFO:teuthology.orchestra.run.smithi148.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T12:13:05.045 INFO:teuthology.orchestra.run.smithi078.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T12:13:05.047 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-21T12:13:05.054 INFO:teuthology.task.internal:Configuring sudo... 2024-09-21T12:13:05.055 DEBUG:teuthology.orchestra.run.smithi078:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-21T12:13:05.089 DEBUG:teuthology.orchestra.run.smithi148:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-21T12:13:05.117 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-21T12:13:05.125 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-21T12:13:05.125 DEBUG:teuthology.orchestra.run.smithi078:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-21T12:13:05.159 DEBUG:teuthology.orchestra.run.smithi148:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-21T12:13:05.176 DEBUG:teuthology.orchestra.run.smithi078:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T12:13:05.260 DEBUG:teuthology.orchestra.run.smithi078:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T12:13:05.316 DEBUG:teuthology.orchestra.run.smithi078:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T12:13:05.401 DEBUG:teuthology.orchestra.run.smithi078:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T12:13:05.445 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T12:13:05.445 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-21T12:13:05.512 DEBUG:teuthology.orchestra.run.smithi148:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T12:13:05.553 DEBUG:teuthology.orchestra.run.smithi148:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T12:13:05.607 DEBUG:teuthology.orchestra.run.smithi148:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T12:13:05.691 DEBUG:teuthology.orchestra.run.smithi148:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T12:13:05.734 DEBUG:teuthology.orchestra.run.smithi148:> set -ex 2024-09-21T12:13:05.734 DEBUG:teuthology.orchestra.run.smithi148:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-21T12:13:05.800 DEBUG:teuthology.orchestra.run.smithi078:> sudo service rsyslog restart 2024-09-21T12:13:05.803 DEBUG:teuthology.orchestra.run.smithi148:> sudo service rsyslog restart 2024-09-21T12:13:05.855 INFO:teuthology.orchestra.run.smithi078.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T12:13:05.890 INFO:teuthology.orchestra.run.smithi148.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T12:13:06.341 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-21T12:13:06.355 INFO:teuthology.task.internal:Starting timer... 2024-09-21T12:13:06.355 INFO:teuthology.run_tasks:Running task pcp... 2024-09-21T12:13:06.376 INFO:teuthology.run_tasks:Running task selinux... 2024-09-21T12:13:06.384 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-09-21T12:13:06.384 DEBUG:teuthology.orchestra.run.smithi078:> sudo service auditd rotate 2024-09-21T12:13:06.436 INFO:teuthology.orchestra.run.smithi078.stdout:Rotating logs: 2024-09-21T12:13:06.439 DEBUG:teuthology.orchestra.run.smithi148:> sudo service auditd rotate 2024-09-21T12:13:06.491 INFO:teuthology.orchestra.run.smithi148.stdout:Rotating logs: 2024-09-21T12:13:06.493 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-21T12:13:06.493 DEBUG:teuthology.orchestra.run.smithi078:> /usr/sbin/getenforce 2024-09-21T12:13:06.548 INFO:teuthology.orchestra.run.smithi078.stdout:Permissive 2024-09-21T12:13:06.548 DEBUG:teuthology.orchestra.run.smithi148:> /usr/sbin/getenforce 2024-09-21T12:13:06.581 INFO:teuthology.orchestra.run.smithi148.stdout:Permissive 2024-09-21T12:13:06.582 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi078.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi148.front.sepia.ceph.com': 'permissive'} 2024-09-21T12:13:06.582 DEBUG:teuthology.orchestra.run.smithi078:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T12:13:06.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T12:13:06.613 DEBUG:teuthology.orchestra.run.smithi148:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T12:13:06.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T12:13:06.650 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-21T12:13:06.650 DEBUG:teuthology.orchestra.run.smithi078:> sudo /usr/sbin/setenforce permissive 2024-09-21T12:13:06.696 DEBUG:teuthology.orchestra.run.smithi148:> sudo /usr/sbin/setenforce permissive 2024-09-21T12:13:06.728 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-21T12:13:06.737 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-21T12:13:06.746 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-21T12:13:06.747 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi078.front.sepia.ceph.com,smithi148.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-21T12:16:51.588 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi078.front.sepia.ceph.com'), Remote(name='ubuntu@smithi148.front.sepia.ceph.com')] 2024-09-21T12:16:51.589 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-21T12:16:51.590 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T12:16:51.666 DEBUG:teuthology.orchestra.run.smithi078:> true 2024-09-21T12:16:51.746 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-09-21T12:16:51.746 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi148.front.sepia.ceph.com' 2024-09-21T12:16:51.747 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi148.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T12:16:51.821 DEBUG:teuthology.orchestra.run.smithi148:> true 2024-09-21T12:16:51.906 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi148.front.sepia.ceph.com' 2024-09-21T12:16:51.906 INFO:teuthology.run_tasks:Running task clock... 2024-09-21T12:16:51.916 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-21T12:16:51.916 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-21T12:16:51.916 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T12:16:51.918 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-21T12:16:51.919 DEBUG:teuthology.orchestra.run.smithi148:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T12:16:51.946 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-21T12:16:51.961 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-21T12:16:51.975 INFO:teuthology.orchestra.run.smithi148.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-21T12:16:51.985 INFO:teuthology.orchestra.run.smithi078.stderr:sudo: ntpd: command not found 2024-09-21T12:16:51.989 INFO:teuthology.orchestra.run.smithi148.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-21T12:16:51.996 INFO:teuthology.orchestra.run.smithi078.stdout:506 Cannot talk to daemon 2024-09-21T12:16:52.009 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-21T12:16:52.013 INFO:teuthology.orchestra.run.smithi148.stderr:sudo: ntpd: command not found 2024-09-21T12:16:52.023 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-21T12:16:52.024 INFO:teuthology.orchestra.run.smithi148.stdout:506 Cannot talk to daemon 2024-09-21T12:16:52.039 INFO:teuthology.orchestra.run.smithi148.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-21T12:16:52.053 INFO:teuthology.orchestra.run.smithi148.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-21T12:16:52.079 INFO:teuthology.orchestra.run.smithi078.stderr:bash: line 1: ntpq: command not found 2024-09-21T12:16:52.082 INFO:teuthology.orchestra.run.smithi078.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T12:16:52.083 INFO:teuthology.orchestra.run.smithi078.stdout:=============================================================================== 2024-09-21T12:16:52.083 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T12:16:52.083 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T12:16:52.083 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T12:16:52.083 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T12:16:52.108 INFO:teuthology.orchestra.run.smithi148.stderr:bash: line 1: ntpq: command not found 2024-09-21T12:16:52.112 INFO:teuthology.orchestra.run.smithi148.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T12:16:52.112 INFO:teuthology.orchestra.run.smithi148.stdout:=============================================================================== 2024-09-21T12:16:52.112 INFO:teuthology.orchestra.run.smithi148.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T12:16:52.112 INFO:teuthology.orchestra.run.smithi148.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T12:16:52.112 INFO:teuthology.orchestra.run.smithi148.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T12:16:52.112 INFO:teuthology.orchestra.run.smithi148.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T12:16:52.113 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-21T12:16:52.219 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'CEPHADM_REFRESH_FAILED'], 'log-only-match': ['CEPHADM_'], 'sha1': '441784946648ff53dd310293f9dd752afeeea72a'} 2024-09-21T12:16:52.219 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-09-21T12:16:52.220 INFO:tasks.cephadm:Cluster fsid is 6263deca-7813-11ef-baf6-efdc52797490 2024-09-21T12:16:52.220 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-21T12:16:52.220 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-09-21T12:16:52.220 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi078': '172.21.15.78', 'mon.smithi148': '172.21.15.148'} 2024-09-21T12:16:52.220 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-21T12:16:52.220 DEBUG:teuthology.orchestra.run.smithi078:> sudo hostname $(hostname -s) 2024-09-21T12:16:52.246 DEBUG:teuthology.orchestra.run.smithi148:> sudo hostname $(hostname -s) 2024-09-21T12:16:52.274 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-09-21T12:16:52.275 DEBUG:teuthology.orchestra.run.smithi078:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-21T12:16:52.645 INFO:teuthology.orchestra.run.smithi078.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 21 12:16 /home/ubuntu/cephtest/cephadm 2024-09-21T12:16:52.645 DEBUG:teuthology.orchestra.run.smithi148:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-21T12:16:52.772 INFO:teuthology.orchestra.run.smithi148.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 21 12:16 /home/ubuntu/cephtest/cephadm 2024-09-21T12:16:52.773 DEBUG:teuthology.orchestra.run.smithi078:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-21T12:16:52.789 DEBUG:teuthology.orchestra.run.smithi148:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-21T12:16:52.813 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-09-21T12:16:52.813 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-21T12:16:52.832 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-21T12:16:54.465 INFO:teuthology.orchestra.run.smithi148.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-21T12:16:54.521 INFO:teuthology.orchestra.run.smithi078.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-21T12:17:06.132 INFO:teuthology.orchestra.run.smithi148.stdout:{ 2024-09-21T12:17:06.133 INFO:teuthology.orchestra.run.smithi148.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-21T12:17:06.133 INFO:teuthology.orchestra.run.smithi148.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-21T12:17:06.133 INFO:teuthology.orchestra.run.smithi148.stdout: "repo_digests": [ 2024-09-21T12:17:06.133 INFO:teuthology.orchestra.run.smithi148.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-21T12:17:06.133 INFO:teuthology.orchestra.run.smithi148.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-21T12:17:06.133 INFO:teuthology.orchestra.run.smithi148.stdout: ] 2024-09-21T12:17:06.133 INFO:teuthology.orchestra.run.smithi148.stdout:} 2024-09-21T12:17:06.410 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:17:06.410 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-21T12:17:06.410 INFO:teuthology.orchestra.run.smithi078.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-21T12:17:06.410 INFO:teuthology.orchestra.run.smithi078.stdout: "repo_digests": [ 2024-09-21T12:17:06.410 INFO:teuthology.orchestra.run.smithi078.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-21T12:17:06.410 INFO:teuthology.orchestra.run.smithi078.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-21T12:17:06.411 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-09-21T12:17:06.411 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:17:06.429 DEBUG:teuthology.orchestra.run.smithi078:> sudo mkdir -p /etc/ceph 2024-09-21T12:17:06.483 DEBUG:teuthology.orchestra.run.smithi148:> sudo mkdir -p /etc/ceph 2024-09-21T12:17:06.511 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 777 /etc/ceph 2024-09-21T12:17:06.536 DEBUG:teuthology.orchestra.run.smithi148:> sudo chmod 777 /etc/ceph 2024-09-21T12:17:06.577 INFO:tasks.cephadm:Writing seed config... 2024-09-21T12:17:06.577 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-21T12:17:06.578 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-21T12:17:06.578 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-21T12:17:06.578 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-21T12:17:06.578 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-21T12:17:06.578 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-21T12:17:06.578 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-21T12:17:06.578 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-21T12:17:06.579 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T12:17:06.579 DEBUG:teuthology.orchestra.run.smithi078:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-21T12:17:06.594 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 6263deca-7813-11ef-baf6-efdc52797490 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-21T12:17:06.594 DEBUG:teuthology.orchestra.run.smithi078:mon.smithi078> sudo journalctl -f -n 0 -u ceph-6263deca-7813-11ef-baf6-efdc52797490@mon.smithi078.service 2024-09-21T12:17:06.637 INFO:tasks.cephadm:Bootstrapping... 2024-09-21T12:17:06.637 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 6263deca-7813-11ef-baf6-efdc52797490 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.78 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-21T12:17:06.823 INFO:teuthology.orchestra.run.smithi078.stderr:-------------------------------------------------------------------------------- 2024-09-21T12:17:06.824 INFO:teuthology.orchestra.run.smithi078.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '6263deca-7813-11ef-baf6-efdc52797490', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.78', '--skip-admin-label'] 2024-09-21T12:17:06.844 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: 5.2.2 2024-09-21T12:17:06.846 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying podman|docker is present... 2024-09-21T12:17:06.866 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: 5.2.2 2024-09-21T12:17:06.868 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying lvm2 is present... 2024-09-21T12:17:06.868 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying time synchronization is in place... 2024-09-21T12:17:06.875 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-21T12:17:06.881 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2024-09-21T12:17:06.888 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: enabled 2024-09-21T12:17:06.894 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: active 2024-09-21T12:17:06.895 INFO:teuthology.orchestra.run.smithi078.stderr:Unit chronyd.service is enabled and running 2024-09-21T12:17:06.895 INFO:teuthology.orchestra.run.smithi078.stderr:Repeating the final host check... 2024-09-21T12:17:06.914 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: 5.2.2 2024-09-21T12:17:06.916 INFO:teuthology.orchestra.run.smithi078.stderr:podman (/bin/podman) version 5.2.2 is present 2024-09-21T12:17:06.916 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl is present 2024-09-21T12:17:06.916 INFO:teuthology.orchestra.run.smithi078.stderr:lvcreate is present 2024-09-21T12:17:06.922 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-21T12:17:06.928 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2024-09-21T12:17:06.935 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: enabled 2024-09-21T12:17:06.941 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: active 2024-09-21T12:17:06.942 INFO:teuthology.orchestra.run.smithi078.stderr:Unit chronyd.service is enabled and running 2024-09-21T12:17:06.942 INFO:teuthology.orchestra.run.smithi078.stderr:Host looks OK 2024-09-21T12:17:06.942 INFO:teuthology.orchestra.run.smithi078.stderr:Cluster fsid: 6263deca-7813-11ef-baf6-efdc52797490 2024-09-21T12:17:06.942 INFO:teuthology.orchestra.run.smithi078.stderr:Acquiring lock 139854496352144 on /run/cephadm/6263deca-7813-11ef-baf6-efdc52797490.lock 2024-09-21T12:17:06.942 INFO:teuthology.orchestra.run.smithi078.stderr:Lock 139854496352144 acquired on /run/cephadm/6263deca-7813-11ef-baf6-efdc52797490.lock 2024-09-21T12:17:06.943 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying IP 172.21.15.78 port 3300 ... 2024-09-21T12:17:06.943 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying IP 172.21.15.78 port 6789 ... 2024-09-21T12:17:06.943 INFO:teuthology.orchestra.run.smithi078.stderr:Base mon IP is 172.21.15.78, final addrv is [v2:172.21.15.78:3300,v1:172.21.15.78:6789] 2024-09-21T12:17:06.945 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.78 metric 100 2024-09-21T12:17:06.945 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.78 metric 100 2024-09-21T12:17:06.948 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-09-21T12:17:06.948 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2024-09-21T12:17:06.948 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2024-09-21T12:17:06.951 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-21T12:17:06.951 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-09-21T12:17:06.951 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-21T12:17:06.951 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-09-21T12:17:06.951 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe8f:d4dd/64 scope link noprefixroute 2024-09-21T12:17:06.951 INFO:teuthology.orchestra.run.smithi078.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-21T12:17:06.952 INFO:teuthology.orchestra.run.smithi078.stderr:Mon IP `172.21.15.78` is in CIDR network `172.21.0.0/20` 2024-09-21T12:17:06.952 INFO:teuthology.orchestra.run.smithi078.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-21T12:17:06.952 INFO:teuthology.orchestra.run.smithi078.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-21T12:17:07.052 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-09-21T12:17:07.387 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Getting image source signatures 2024-09-21T12:17:07.387 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-09-21T12:17:07.387 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-09-21T12:17:07.387 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-09-21T12:17:07.387 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-09-21T12:17:07.388 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-09-21T12:17:07.388 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-21T12:17:07.547 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: Writing manifest to image destination 2024-09-21T12:17:07.707 INFO:teuthology.orchestra.run.smithi078.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-21T12:17:08.134 INFO:teuthology.orchestra.run.smithi078.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-21T12:17:08.665 INFO:teuthology.orchestra.run.smithi078.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-21T12:17:08.665 INFO:teuthology.orchestra.run.smithi078.stderr:Extracting ceph user uid/gid from container image... 2024-09-21T12:17:09.039 INFO:teuthology.orchestra.run.smithi078.stderr:stat: 167 167 2024-09-21T12:17:09.872 INFO:teuthology.orchestra.run.smithi078.stderr:Creating initial keys... 2024-09-21T12:17:13.676 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-authtool: AQBIue5mT194JRAAN5Rdr954C7tDmKbraKbpJg== 2024-09-21T12:17:21.355 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-authtool: AQBRue5mX/DLDhAALcYymwjhO7Qm16R0IowNqw== 2024-09-21T12:17:22.319 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph-authtool: AQBSue5mOjMDDxAA3ybYu1MYtwNfH9TrgrBF5Q== 2024-09-21T12:17:22.801 INFO:teuthology.orchestra.run.smithi078.stderr:Creating initial monmap... 2024-09-21T12:17:23.208 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-21T12:17:23.208 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-09-21T12:17:23.208 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 6263deca-7813-11ef-baf6-efdc52797490 2024-09-21T12:17:23.209 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-21T12:17:23.731 INFO:teuthology.orchestra.run.smithi078.stderr:monmaptool for smithi078 [v2:172.21.15.78:3300,v1:172.21.15.78:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-21T12:17:23.731 INFO:teuthology.orchestra.run.smithi078.stderr:setting min_mon_release = octopus 2024-09-21T12:17:23.732 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: set fsid to 6263deca-7813-11ef-baf6-efdc52797490 2024-09-21T12:17:23.732 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-21T12:17:23.732 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-21T12:17:23.732 INFO:teuthology.orchestra.run.smithi078.stderr:Creating mon... 2024-09-21T12:17:24.663 INFO:teuthology.orchestra.run.smithi078.stderr:create mon.smithi078 on 2024-09-21T12:17:24.898 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-21T12:17:25.047 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-6263deca-7813-11ef-baf6-efdc52797490.target → /etc/systemd/system/ceph-6263deca-7813-11ef-baf6-efdc52797490.target. 2024-09-21T12:17:25.048 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-6263deca-7813-11ef-baf6-efdc52797490.target → /etc/systemd/system/ceph-6263deca-7813-11ef-baf6-efdc52797490.target. 2024-09-21T12:17:25.346 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Failed to reset failed state of unit ceph-6263deca-7813-11ef-baf6-efdc52797490@mon.smithi078.service: Unit ceph-6263deca-7813-11ef-baf6-efdc52797490@mon.smithi078.service not loaded. 2024-09-21T12:17:25.353 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Created symlink /etc/systemd/system/ceph-6263deca-7813-11ef-baf6-efdc52797490.target.wants/ceph-6263deca-7813-11ef-baf6-efdc52797490@mon.smithi078.service → /etc/systemd/system/ceph-6263deca-7813-11ef-baf6-efdc52797490@.service. 2024-09-21T12:17:26.026 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:25 smithi078 podman[23326]: 2024-09-21 12:17:25.98405103 +0000 UTC m=+0.346244410 container start 6989c8855aad27cfa9971b69c30587c905d52428c1a5ef16c2383957e2465f53 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi078, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, vendor=Red Hat, Inc., io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, com.redhat.component=centos-stream-container, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754) 2024-09-21T12:17:26.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:26 smithi078 ceph-mon[23345]: mkfs 6263deca-7813-11ef-baf6-efdc52797490 2024-09-21T12:17:26.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:26 smithi078 ceph-mon[23345]: mon.smithi078 is new leader, mons smithi078 in quorum (ranks 0) 2024-09-21T12:17:26.946 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: disabled 2024-09-21T12:17:26.953 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2024-09-21T12:17:26.953 INFO:teuthology.orchestra.run.smithi078.stderr:firewalld.service is not enabled 2024-09-21T12:17:26.953 INFO:teuthology.orchestra.run.smithi078.stderr:Not possible to enable service . firewalld.service is not available 2024-09-21T12:17:26.954 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mon to start... 2024-09-21T12:17:26.954 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mon... 2024-09-21T12:17:27.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:26 smithi078 bash[23326]: 6989c8855aad27cfa9971b69c30587c905d52428c1a5ef16c2383957e2465f53 2024-09-21T12:17:27.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:26 smithi078 systemd[1]: Started Ceph mon.smithi078 for 6263deca-7813-11ef-baf6-efdc52797490. 2024-09-21T12:17:27.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:27 smithi078 ceph-mon[23345]: mon.smithi078 is new leader, mons smithi078 in quorum (ranks 0) 2024-09-21T12:17:27.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:27 smithi078 ceph-mon[23345]: monmap e1: 1 mons at {smithi078=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0]} 2024-09-21T12:17:27.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:27 smithi078 ceph-mon[23345]: fsmap 2024-09-21T12:17:27.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:27 smithi078 ceph-mon[23345]: osdmap e1: 0 total, 0 up, 0 in 2024-09-21T12:17:27.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:27 smithi078 ceph-mon[23345]: mgrmap e1: no daemons active 2024-09-21T12:17:27.839 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: cluster: 2024-09-21T12:17:27.839 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: id: 6263deca-7813-11ef-baf6-efdc52797490 2024-09-21T12:17:27.839 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-09-21T12:17:27.839 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-21T12:17:27.839 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: services: 2024-09-21T12:17:27.839 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi078 (age 1.81547s) 2024-09-21T12:17:27.839 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mgr: no daemons active 2024-09-21T12:17:27.839 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-09-21T12:17:27.840 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-21T12:17:27.840 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: data: 2024-09-21T12:17:27.840 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-09-21T12:17:27.840 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-09-21T12:17:27.840 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-09-21T12:17:27.840 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: pgs: 2024-09-21T12:17:27.840 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-21T12:17:28.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:28 smithi078 ceph-mon[23345]: from='client.? 172.21.15.78:0/3253694378' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-21T12:17:28.373 INFO:teuthology.orchestra.run.smithi078.stderr:mon is available 2024-09-21T12:17:28.373 INFO:teuthology.orchestra.run.smithi078.stderr:Assimilating anything we can from ceph.conf... 2024-09-21T12:17:28.906 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-21T12:17:28.907 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [global] 2024-09-21T12:17:28.907 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: fsid = 6263deca-7813-11ef-baf6-efdc52797490 2024-09-21T12:17:28.907 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.78:3300,v1:172.21.15.78:6789] 2024-09-21T12:17:28.907 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-21T12:17:28.907 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-21T12:17:28.907 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-21T12:17:28.907 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-21T12:17:28.907 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-21T12:17:28.907 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [mgr] 2024-09-21T12:17:28.907 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-21T12:17:28.907 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-21T12:17:28.907 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [osd] 2024-09-21T12:17:28.908 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-21T12:17:28.908 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-21T12:17:29.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:29 smithi078 ceph-mon[23345]: from='client.? 172.21.15.78:0/1153971894' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-21T12:17:29.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:29 smithi078 ceph-mon[23345]: from='client.? 172.21.15.78:0/1153971894' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-21T12:17:29.429 INFO:teuthology.orchestra.run.smithi078.stderr:Generating new minimal ceph.conf... 2024-09-21T12:17:30.299 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:30 smithi078 ceph-mon[23345]: from='client.? 172.21.15.78:0/2513190078' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:17:30.569 INFO:teuthology.orchestra.run.smithi078.stderr:Restarting the monitor... 2024-09-21T12:17:30.911 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:30 smithi078 systemd[1]: Stopping Ceph mon.smithi078 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:17:30.912 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:30 smithi078 bash[23563]: Error: no container with name or ID "ceph-6263deca-7813-11ef-baf6-efdc52797490-mon.smithi078" found: no such container 2024-09-21T12:17:30.912 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:30 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi078[23341]: 2024-09-21T12:17:30.725+0000 7f80027f6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi078 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T12:17:30.912 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:30 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi078[23341]: 2024-09-21T12:17:30.725+0000 7f80027f6700 -1 mon.smithi078@0(leader) e1 *** Got Signal Terminated *** 2024-09-21T12:17:31.256 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:30 smithi078 podman[23574]: 2024-09-21 12:17:30.910534009 +0000 UTC m=+0.285582358 container died 6989c8855aad27cfa9971b69c30587c905d52428c1a5ef16c2383957e2465f53 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi078, io.openshift.expose-services=, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, ceph=True, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, vcs-type=git, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, release=754, distribution-scope=public, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8) 2024-09-21T12:17:31.537 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:31 smithi078 podman[23574]: 2024-09-21 12:17:31.334554448 +0000 UTC m=+0.709602803 container cleanup 6989c8855aad27cfa9971b69c30587c905d52428c1a5ef16c2383957e2465f53 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi078, architecture=x86_64, ceph=True, name=centos-stream, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, RELEASE=HEAD, GIT_CLEAN=True, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, build-date=2022-05-03T08:36:31.336870) 2024-09-21T12:17:31.537 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:31 smithi078 bash[23574]: ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi078 2024-09-21T12:17:31.537 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:31 smithi078 bash[23599]: Error: no container with name or ID "ceph-6263deca-7813-11ef-baf6-efdc52797490-mon.smithi078" found: no such container 2024-09-21T12:17:31.787 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:31 smithi078 podman[23586]: 2024-09-21 12:17:31.535348216 +0000 UTC m=+0.616211507 container remove 6989c8855aad27cfa9971b69c30587c905d52428c1a5ef16c2383957e2465f53 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi078, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, version=8, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , vcs-type=git, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, architecture=x86_64, io.buildah.version=1.19.8) 2024-09-21T12:17:31.787 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:31 smithi078 systemd[1]: ceph-6263deca-7813-11ef-baf6-efdc52797490@mon.smithi078.service: Deactivated successfully. 2024-09-21T12:17:31.787 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:31 smithi078 systemd[1]: Stopped Ceph mon.smithi078 for 6263deca-7813-11ef-baf6-efdc52797490. 2024-09-21T12:17:31.787 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:31 smithi078 systemd[1]: Starting Ceph mon.smithi078 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:17:31.787 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:31 smithi078 podman[23662]: 2024-09-21 12:17:31.673641353 +0000 UTC m=+0.016319311 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-21T12:17:31.787 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:31 smithi078 podman[23662]: 2024-09-21 12:17:31.786040706 +0000 UTC m=+0.128718669 container create 8261b0b2c7e041643c0685bc38c800cc93b33e816d6f9c3f3bdf60dc8d0b8830 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi078, release=754, GIT_CLEAN=True, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, vcs-type=git, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, version=8, ceph=True, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, architecture=x86_64, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-21T12:17:32.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:31 smithi078 podman[23662]: 2024-09-21 12:17:31.997292235 +0000 UTC m=+0.339970191 container init 8261b0b2c7e041643c0685bc38c800cc93b33e816d6f9c3f3bdf60dc8d0b8830 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi078, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, distribution-scope=public, maintainer=Guillaume Abrioux , architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, version=8, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., GIT_CLEAN=True, RELEASE=HEAD, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=) 2024-09-21T12:17:32.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 podman[23662]: 2024-09-21 12:17:32.000391854 +0000 UTC m=+0.343069823 container start 8261b0b2c7e041643c0685bc38c800cc93b33e816d6f9c3f3bdf60dc8d0b8830 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi078, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , version=8, vcs-type=git, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, GIT_CLEAN=True, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, io.buildah.version=1.19.8) 2024-09-21T12:17:32.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T12:17:32.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-21T12:17:32.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: pidfile_write: ignore empty --pid-file 2024-09-21T12:17:32.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: load: jerasure load: lrc 2024-09-21T12:17:32.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: RocksDB version: 6.15.5 2024-09-21T12:17:32.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-21T12:17:32.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Compile date Apr 18 2022 2024-09-21T12:17:32.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: DB SUMMARY 2024-09-21T12:17:32.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: DB Session ID: I9S6Z65Z5DH7MXH3DGJU 2024-09-21T12:17:32.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: CURRENT file: CURRENT 2024-09-21T12:17:32.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T12:17:32.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-21T12:17:32.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi078/store.db dir, Total Num: 1, files: 000008.sst 2024-09-21T12:17:32.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi078/store.db: 000010.log size: 83010 ; 2024-09-21T12:17:32.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.error_if_exists: 0 2024-09-21T12:17:32.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.create_if_missing: 0 2024-09-21T12:17:32.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.paranoid_checks: 1 2024-09-21T12:17:32.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T12:17:32.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.env: 0x55ee65ec6860 2024-09-21T12:17:32.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.fs: Posix File System 2024-09-21T12:17:32.059 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.info_log: 0x55ee674e7fa0 2024-09-21T12:17:32.059 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T12:17:32.059 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.statistics: (nil) 2024-09-21T12:17:32.059 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.use_fsync: 0 2024-09-21T12:17:32.059 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.max_log_file_size: 0 2024-09-21T12:17:32.059 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T12:17:32.059 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T12:17:32.059 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T12:17:32.059 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T12:17:32.059 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.allow_fallocate: 1 2024-09-21T12:17:32.059 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T12:17:32.059 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T12:17:32.059 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.use_direct_reads: 0 2024-09-21T12:17:32.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T12:17:32.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T12:17:32.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.db_log_dir: 2024-09-21T12:17:32.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi078/store.db 2024-09-21T12:17:32.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T12:17:32.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T12:17:32.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T12:17:32.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T12:17:32.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T12:17:32.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T12:17:32.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T12:17:32.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T12:17:32.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.write_buffer_manager: 0x55ee675d8330 2024-09-21T12:17:32.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T12:17:32.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-21T12:17:32.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T12:17:32.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T12:17:32.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T12:17:32.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T12:17:32.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T12:17:32.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T12:17:32.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T12:17:32.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.unordered_write: 0 2024-09-21T12:17:32.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T12:17:32.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T12:17:32.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T12:17:32.062 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T12:17:32.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.row_cache: None 2024-09-21T12:17:32.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.wal_filter: None 2024-09-21T12:17:32.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T12:17:32.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T12:17:32.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.preserve_deletes: 0 2024-09-21T12:17:32.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.two_write_queues: 0 2024-09-21T12:17:32.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T12:17:32.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.atomic_flush: 0 2024-09-21T12:17:32.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T12:17:32.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T12:17:32.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T12:17:32.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.log_readahead_size: 0 2024-09-21T12:17:32.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T12:17:32.063 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T12:17:32.064 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T12:17:32.064 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T12:17:32.064 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T12:17:32.064 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T12:17:32.064 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.max_background_jobs: 2 2024-09-21T12:17:32.064 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.max_background_compactions: -1 2024-09-21T12:17:32.064 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.max_subcompactions: 1 2024-09-21T12:17:32.064 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T12:17:32.064 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T12:17:32.064 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T12:17:32.064 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T12:17:32.064 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T12:17:32.064 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T12:17:32.064 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T12:17:32.064 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T12:17:32.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.max_open_files: -1 2024-09-21T12:17:32.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T12:17:32.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T12:17:32.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T12:17:32.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T12:17:32.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.max_background_flushes: -1 2024-09-21T12:17:32.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Compression algorithms supported: 2024-09-21T12:17:32.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T12:17:32.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: kZSTD supported: 0 2024-09-21T12:17:32.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: kXpressCompression supported: 0 2024-09-21T12:17:32.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T12:17:32.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: kLZ4Compression supported: 1 2024-09-21T12:17:32.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: kBZip2Compression supported: 0 2024-09-21T12:17:32.065 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: kZlibCompression supported: 1 2024-09-21T12:17:32.066 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: kSnappyCompression supported: 1 2024-09-21T12:17:32.066 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T12:17:32.066 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi078/store.db/MANIFEST-000009 2024-09-21T12:17:32.066 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-21T12:17:32.066 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T12:17:32.066 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.merge_operator: 2024-09-21T12:17:32.066 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.compaction_filter: None 2024-09-21T12:17:32.066 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T12:17:32.066 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T12:17:32.066 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T12:17:32.066 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T12:17:32.066 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ee674b3d00) 2024-09-21T12:17:32.066 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: cache_index_and_filter_blocks: 1 2024-09-21T12:17:32.066 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T12:17:32.067 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T12:17:32.067 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: pin_top_level_index_and_filter: 1 2024-09-21T12:17:32.067 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: index_type: 0 2024-09-21T12:17:32.067 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: data_block_index_type: 0 2024-09-21T12:17:32.067 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: index_shortening: 1 2024-09-21T12:17:32.067 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T12:17:32.067 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: hash_index_allow_collision: 1 2024-09-21T12:17:32.067 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: checksum: 1 2024-09-21T12:17:32.067 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: no_block_cache: 0 2024-09-21T12:17:32.067 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache: 0x55ee6751e170 2024-09-21T12:17:32.067 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache_name: BinnedLRUCache 2024-09-21T12:17:32.067 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache_options: 2024-09-21T12:17:32.067 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: capacity : 536870912 2024-09-21T12:17:32.067 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: num_shard_bits : 4 2024-09-21T12:17:32.068 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: strict_capacity_limit : 0 2024-09-21T12:17:32.068 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: high_pri_pool_ratio: 0.000 2024-09-21T12:17:32.068 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache_compressed: (nil) 2024-09-21T12:17:32.068 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: persistent_cache: (nil) 2024-09-21T12:17:32.068 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_size: 4096 2024-09-21T12:17:32.068 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_size_deviation: 10 2024-09-21T12:17:32.068 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_restart_interval: 16 2024-09-21T12:17:32.068 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: index_block_restart_interval: 1 2024-09-21T12:17:32.068 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: metadata_block_size: 4096 2024-09-21T12:17:32.068 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: partition_filters: 0 2024-09-21T12:17:32.068 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: use_delta_encoding: 1 2024-09-21T12:17:32.068 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-21T12:17:32.068 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: whole_key_filtering: 1 2024-09-21T12:17:32.068 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: verify_compression: 0 2024-09-21T12:17:32.069 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: read_amp_bytes_per_bit: 0 2024-09-21T12:17:32.069 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: format_version: 4 2024-09-21T12:17:32.069 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: enable_index_compression: 1 2024-09-21T12:17:32.069 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_align: 0 2024-09-21T12:17:32.069 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T12:17:32.069 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T12:17:32.069 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.compression: NoCompression 2024-09-21T12:17:32.069 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T12:17:32.069 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T12:17:32.069 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T12:17:32.069 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.num_levels: 7 2024-09-21T12:17:32.069 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T12:17:32.069 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T12:17:32.069 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T12:17:32.070 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T12:17:32.070 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T12:17:32.070 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T12:17:32.070 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T12:17:32.070 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T12:17:32.070 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T12:17:32.070 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T12:17:32.070 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T12:17:32.070 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T12:17:32.070 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T12:17:32.070 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T12:17:32.071 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T12:17:32.071 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T12:17:32.071 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T12:17:32.071 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T12:17:32.071 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T12:17:32.071 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T12:17:32.071 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T12:17:32.073 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T12:17:32.073 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T12:17:32.073 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T12:17:32.073 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T12:17:32.073 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T12:17:32.073 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T12:17:32.073 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T12:17:32.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T12:17:32.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T12:17:32.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T12:17:32.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T12:17:32.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T12:17:32.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T12:17:32.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.arena_block_size: 4194304 2024-09-21T12:17:32.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T12:17:32.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T12:17:32.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-21T12:17:32.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T12:17:32.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T12:17:32.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T12:17:32.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T12:17:32.075 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T12:17:32.075 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T12:17:32.075 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T12:17:32.075 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T12:17:32.075 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T12:17:32.075 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T12:17:32.075 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T12:17:32.075 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.table_properties_collectors: 2024-09-21T12:17:32.075 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.inplace_update_support: 0 2024-09-21T12:17:32.075 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T12:17:32.075 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T12:17:32.075 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T12:17:32.075 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T12:17:32.076 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.bloom_locality: 0 2024-09-21T12:17:32.076 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.max_successive_merges: 0 2024-09-21T12:17:32.076 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T12:17:32.076 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T12:17:32.076 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T12:17:32.076 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T12:17:32.076 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.ttl: 2592000 2024-09-21T12:17:32.076 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T12:17:32.076 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.enable_blob_files: false 2024-09-21T12:17:32.076 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.min_blob_size: 0 2024-09-21T12:17:32.076 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T12:17:32.076 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T12:17:32.076 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T12:17:32.076 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T12:17:32.077 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi078/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T12:17:32.077 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-09-21T12:17:32.077 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-09-21T12:17:32.077 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726921052026170, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-21T12:17:32.077 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-09-21T12:17:32.077 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-21T12:17:32.077 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726921052026862, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79544, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77808, "index_size": 224, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9914, "raw_average_key_size": 46, "raw_value_size": 72091, "raw_average_value_size": 341, "num_data_blocks": 10, "num_entries": 211, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726921052, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "5f1fcf2f-3390-4361-b4df-a0067ec09e39", "db_session_id": "I9S6Z65Z5DH7MXH3DGJU"}} 2024-09-21T12:17:32.077 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-09-21T12:17:32.077 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726921052027173, "job": 1, "event": "recovery_finished"} 2024-09-21T12:17:32.077 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi078/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T12:17:32.077 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55ee67504a80 2024-09-21T12:17:32.077 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: DB pointer 0x55ee67514000 2024-09-21T12:17:32.077 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-21T12:17:32.077 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-21T12:17:32.077 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** DB Stats ** 2024-09-21T12:17:32.077 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T12:17:32.078 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T12:17:32.078 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T12:17:32.078 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T12:17:32.078 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T12:17:32.078 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-21T12:17:32.078 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T12:17:32.078 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-21T12:17:32.078 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** Compaction Stats [default] ** 2024-09-21T12:17:32.078 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T12:17:32.078 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T12:17:32.078 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: L0 2/0 79.48 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 164.2 0.00 0.00 1 0.000 0 0 2024-09-21T12:17:32.078 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Sum 2/0 79.48 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 164.2 0.00 0.00 1 0.000 0 0 2024-09-21T12:17:32.078 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 164.2 0.00 0.00 1 0.000 0 0 2024-09-21T12:17:32.078 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-21T12:17:32.078 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** Compaction Stats [default] ** 2024-09-21T12:17:32.079 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T12:17:32.079 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T12:17:32.079 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 164.2 0.00 0.00 1 0.000 0 0 2024-09-21T12:17:32.079 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T12:17:32.079 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-21T12:17:32.079 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T12:17:32.079 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T12:17:32.079 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T12:17:32.079 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T12:17:32.079 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative compaction: 0.00 GB write, 26.12 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T12:17:32.079 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval compaction: 0.00 GB write, 26.12 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T12:17:32.079 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T12:17:32.079 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-21T12:17:32.079 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T12:17:32.080 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-21T12:17:32.080 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** Compaction Stats [default] ** 2024-09-21T12:17:32.080 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T12:17:32.080 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T12:17:32.080 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: L0 2/0 79.48 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 164.2 0.00 0.00 1 0.000 0 0 2024-09-21T12:17:32.080 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Sum 2/0 79.48 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 164.2 0.00 0.00 1 0.000 0 0 2024-09-21T12:17:32.080 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-21T12:17:32.080 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-21T12:17:32.080 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** Compaction Stats [default] ** 2024-09-21T12:17:32.080 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T12:17:32.080 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T12:17:32.080 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 164.2 0.00 0.00 1 0.000 0 0 2024-09-21T12:17:32.080 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T12:17:32.080 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-21T12:17:32.080 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T12:17:32.081 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T12:17:32.081 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T12:17:32.081 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T12:17:32.081 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative compaction: 0.00 GB write, 25.83 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T12:17:32.081 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T12:17:32.081 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T12:17:32.081 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-21T12:17:32.081 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T12:17:32.081 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: starting mon.smithi078 rank 0 at public addrs [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] at bind addrs [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi078 fsid 6263deca-7813-11ef-baf6-efdc52797490 2024-09-21T12:17:32.081 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: mon.smithi078@-1(???) e1 preinit fsid 6263deca-7813-11ef-baf6-efdc52797490 2024-09-21T12:17:32.081 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: mon.smithi078@-1(???).mds e1 new map 2024-09-21T12:17:32.081 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: mon.smithi078@-1(???).mds e1 print_map 2024-09-21T12:17:32.081 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: e1 2024-09-21T12:17:32.081 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T12:17:32.081 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T12:17:32.081 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: legacy client fscid: -1 2024-09-21T12:17:32.082 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-21T12:17:32.082 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: No filesystems configured 2024-09-21T12:17:32.082 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: mon.smithi078@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-21T12:17:32.082 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: mon.smithi078@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T12:17:32.082 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: mon.smithi078@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T12:17:32.082 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: mon.smithi078@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T12:17:32.082 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: mon.smithi078@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-21T12:17:32.082 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: expand_channel_meta expand map: {default=false} 2024-09-21T12:17:32.082 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: expand_channel_meta from 'false' to 'false' 2024-09-21T12:17:32.082 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: expand_channel_meta expanded map: {default=false} 2024-09-21T12:17:32.082 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: expand_channel_meta expand map: {default=info} 2024-09-21T12:17:32.082 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: expand_channel_meta from 'info' to 'info' 2024-09-21T12:17:32.082 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: expand_channel_meta expanded map: {default=info} 2024-09-21T12:17:32.082 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: expand_channel_meta expand map: {default=daemon} 2024-09-21T12:17:32.082 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: expand_channel_meta from 'daemon' to 'daemon' 2024-09-21T12:17:32.082 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: expand_channel_meta expanded map: {default=daemon} 2024-09-21T12:17:32.083 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: expand_channel_meta expand map: {default=debug} 2024-09-21T12:17:32.083 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: expand_channel_meta from 'debug' to 'debug' 2024-09-21T12:17:32.083 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: expand_channel_meta expanded map: {default=debug} 2024-09-21T12:17:32.101 INFO:teuthology.orchestra.run.smithi078.stderr:Setting mon public_network to 172.21.0.0/20 2024-09-21T12:17:32.345 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: mon.smithi078 is new leader, mons smithi078 in quorum (ranks 0) 2024-09-21T12:17:32.345 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: monmap e1: 1 mons at {smithi078=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0]} 2024-09-21T12:17:32.345 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: fsmap 2024-09-21T12:17:32.345 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: osdmap e1: 0 total, 0 up, 0 in 2024-09-21T12:17:32.345 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 ceph-mon[23681]: mgrmap e1: no daemons active 2024-09-21T12:17:32.345 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 bash[23662]: 8261b0b2c7e041643c0685bc38c800cc93b33e816d6f9c3f3bdf60dc8d0b8830 2024-09-21T12:17:32.345 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:32 smithi078 systemd[1]: Started Ceph mon.smithi078 for 6263deca-7813-11ef-baf6-efdc52797490. 2024-09-21T12:17:33.123 INFO:teuthology.orchestra.run.smithi078.stderr:Wrote config to /etc/ceph/ceph.conf 2024-09-21T12:17:33.123 INFO:teuthology.orchestra.run.smithi078.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-21T12:17:33.123 INFO:teuthology.orchestra.run.smithi078.stderr:Creating mgr... 2024-09-21T12:17:33.124 INFO:teuthology.orchestra.run.smithi078.stderr:Verifying port 9283 ... 2024-09-21T12:17:33.301 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Failed to reset failed state of unit ceph-6263deca-7813-11ef-baf6-efdc52797490@mgr.smithi078.ntwyan.service: Unit ceph-6263deca-7813-11ef-baf6-efdc52797490@mgr.smithi078.ntwyan.service not loaded. 2024-09-21T12:17:33.308 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: Created symlink /etc/systemd/system/ceph-6263deca-7813-11ef-baf6-efdc52797490.target.wants/ceph-6263deca-7813-11ef-baf6-efdc52797490@mgr.smithi078.ntwyan.service → /etc/systemd/system/ceph-6263deca-7813-11ef-baf6-efdc52797490@.service. 2024-09-21T12:17:33.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:33 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/1317132956' entity='client.admin' 2024-09-21T12:17:34.004 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: disabled 2024-09-21T12:17:34.011 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2024-09-21T12:17:34.012 INFO:teuthology.orchestra.run.smithi078.stderr:firewalld.service is not enabled 2024-09-21T12:17:34.012 INFO:teuthology.orchestra.run.smithi078.stderr:Not possible to enable service . firewalld.service is not available 2024-09-21T12:17:34.020 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: disabled 2024-09-21T12:17:34.027 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2024-09-21T12:17:34.028 INFO:teuthology.orchestra.run.smithi078.stderr:firewalld.service is not enabled 2024-09-21T12:17:34.028 INFO:teuthology.orchestra.run.smithi078.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-09-21T12:17:34.028 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mgr to start... 2024-09-21T12:17:34.028 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mgr... 2024-09-21T12:17:34.561 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-21T12:17:34.561 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-09-21T12:17:34.561 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsid": "6263deca-7813-11ef-baf6-efdc52797490", 2024-09-21T12:17:34.561 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "health": { 2024-09-21T12:17:34.561 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-21T12:17:34.562 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "checks": {}, 2024-09-21T12:17:34.562 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mutes": [] 2024-09-21T12:17:34.562 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-21T12:17:34.562 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-21T12:17:34.562 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum": [ 2024-09-21T12:17:34.562 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 0 2024-09-21T12:17:34.562 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-21T12:17:34.562 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-21T12:17:34.562 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "smithi078" 2024-09-21T12:17:34.562 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-21T12:17:34.562 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-09-21T12:17:34.562 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "monmap": { 2024-09-21T12:17:34.562 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T12:17:34.563 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-21T12:17:34.563 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-21T12:17:34.563 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-21T12:17:34.563 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osdmap": { 2024-09-21T12:17:34.563 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T12:17:34.563 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-21T12:17:34.563 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-21T12:17:34.563 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-21T12:17:34.563 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-21T12:17:34.566 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-21T12:17:34.566 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-21T12:17:34.566 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-21T12:17:34.567 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgmap": { 2024-09-21T12:17:34.567 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-21T12:17:34.567 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-21T12:17:34.567 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-21T12:17:34.567 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-21T12:17:34.567 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-21T12:17:34.567 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-21T12:17:34.567 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-21T12:17:34.567 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-21T12:17:34.567 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-21T12:17:34.567 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsmap": { 2024-09-21T12:17:34.567 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T12:17:34.567 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-21T12:17:34.568 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-21T12:17:34.568 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-21T12:17:34.568 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-21T12:17:34.568 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": false, 2024-09-21T12:17:34.568 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-21T12:17:34.568 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modules": [ 2024-09-21T12:17:34.568 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "iostat", 2024-09-21T12:17:34.568 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "nfs", 2024-09-21T12:17:34.568 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "restful" 2024-09-21T12:17:34.568 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-21T12:17:34.568 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-09-21T12:17:34.568 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-21T12:17:34.568 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "servicemap": { 2024-09-21T12:17:34.568 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T12:17:34.568 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modified": "2024-09-21T12:17:26.021949+0000", 2024-09-21T12:17:34.569 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-09-21T12:17:34.569 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-21T12:17:34.569 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-21T12:17:34.569 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-09-21T12:17:34.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:34 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/775007179' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-21T12:17:35.167 INFO:teuthology.orchestra.run.smithi078.stderr:mgr not available, waiting (1/15)... 2024-09-21T12:17:37.626 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-21T12:17:37.626 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-09-21T12:17:37.627 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsid": "6263deca-7813-11ef-baf6-efdc52797490", 2024-09-21T12:17:37.627 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "health": { 2024-09-21T12:17:37.627 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-21T12:17:37.627 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "checks": {}, 2024-09-21T12:17:37.627 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mutes": [] 2024-09-21T12:17:37.627 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-21T12:17:37.627 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-21T12:17:37.627 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum": [ 2024-09-21T12:17:37.627 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 0 2024-09-21T12:17:37.627 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-21T12:17:37.627 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-21T12:17:37.627 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "smithi078" 2024-09-21T12:17:37.628 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-21T12:17:37.628 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_age": 5, 2024-09-21T12:17:37.628 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "monmap": { 2024-09-21T12:17:37.628 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T12:17:37.628 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-21T12:17:37.628 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-21T12:17:37.628 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-21T12:17:37.628 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osdmap": { 2024-09-21T12:17:37.628 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T12:17:37.628 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-21T12:17:37.628 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-21T12:17:37.628 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-21T12:17:37.628 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-21T12:17:37.632 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-21T12:17:37.632 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-21T12:17:37.632 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-21T12:17:37.632 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgmap": { 2024-09-21T12:17:37.632 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-21T12:17:37.632 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-21T12:17:37.632 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-21T12:17:37.632 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-21T12:17:37.632 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-21T12:17:37.632 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-21T12:17:37.632 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-21T12:17:37.633 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-21T12:17:37.633 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-21T12:17:37.633 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsmap": { 2024-09-21T12:17:37.633 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T12:17:37.633 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-21T12:17:37.633 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-21T12:17:37.633 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-21T12:17:37.633 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-21T12:17:37.633 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": false, 2024-09-21T12:17:37.633 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-21T12:17:37.633 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modules": [ 2024-09-21T12:17:37.634 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "iostat", 2024-09-21T12:17:37.634 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "nfs", 2024-09-21T12:17:37.634 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "restful" 2024-09-21T12:17:37.634 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-21T12:17:37.634 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-09-21T12:17:37.634 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-21T12:17:37.634 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "servicemap": { 2024-09-21T12:17:37.634 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T12:17:37.634 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modified": "2024-09-21T12:17:26.021949+0000", 2024-09-21T12:17:37.634 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-09-21T12:17:37.634 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-21T12:17:37.634 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-21T12:17:37.634 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-09-21T12:17:38.154 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:38 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/2779670269' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-21T12:17:38.280 INFO:teuthology.orchestra.run.smithi078.stderr:mgr not available, waiting (2/15)... 2024-09-21T12:17:40.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:40 smithi078 ceph-mon[23681]: Activating manager daemon smithi078.ntwyan 2024-09-21T12:17:40.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:40 smithi078 ceph-mon[23681]: mgrmap e2: smithi078.ntwyan(active, starting, since 0.00280388s) 2024-09-21T12:17:40.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:40 smithi078 ceph-mon[23681]: from='mgr.14100 172.21.15.78:0/2417927272' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T12:17:40.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:40 smithi078 ceph-mon[23681]: from='mgr.14100 172.21.15.78:0/2417927272' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T12:17:40.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:40 smithi078 ceph-mon[23681]: from='mgr.14100 172.21.15.78:0/2417927272' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T12:17:40.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:40 smithi078 ceph-mon[23681]: from='mgr.14100 172.21.15.78:0/2417927272' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-21T12:17:40.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:40 smithi078 ceph-mon[23681]: from='mgr.14100 172.21.15.78:0/2417927272' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mgr metadata", "who": "smithi078.ntwyan", "id": "smithi078.ntwyan"}]: dispatch 2024-09-21T12:17:40.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:40 smithi078 ceph-mon[23681]: Manager daemon smithi078.ntwyan is now available 2024-09-21T12:17:40.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:40 smithi078 ceph-mon[23681]: from='mgr.14100 172.21.15.78:0/2417927272' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/mirror_snapshot_schedule"}]: dispatch 2024-09-21T12:17:40.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:40 smithi078 ceph-mon[23681]: from='mgr.14100 172.21.15.78:0/2417927272' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/trash_purge_schedule"}]: dispatch 2024-09-21T12:17:40.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:40 smithi078 ceph-mon[23681]: from='mgr.14100 172.21.15.78:0/2417927272' entity='mgr.smithi078.ntwyan' 2024-09-21T12:17:40.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:40 smithi078 ceph-mon[23681]: from='mgr.14100 172.21.15.78:0/2417927272' entity='mgr.smithi078.ntwyan' 2024-09-21T12:17:40.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:40 smithi078 ceph-mon[23681]: from='mgr.14100 172.21.15.78:0/2417927272' entity='mgr.smithi078.ntwyan' 2024-09-21T12:17:40.803 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-21T12:17:40.803 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-09-21T12:17:40.803 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsid": "6263deca-7813-11ef-baf6-efdc52797490", 2024-09-21T12:17:40.803 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "health": { 2024-09-21T12:17:40.803 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-21T12:17:40.803 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "checks": {}, 2024-09-21T12:17:40.803 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mutes": [] 2024-09-21T12:17:40.804 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-21T12:17:40.804 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-21T12:17:40.804 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum": [ 2024-09-21T12:17:40.804 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 0 2024-09-21T12:17:40.804 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-21T12:17:40.804 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-21T12:17:40.804 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "smithi078" 2024-09-21T12:17:40.804 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-21T12:17:40.804 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_age": 8, 2024-09-21T12:17:40.804 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "monmap": { 2024-09-21T12:17:40.804 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T12:17:40.804 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-21T12:17:40.804 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-21T12:17:40.804 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-21T12:17:40.804 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osdmap": { 2024-09-21T12:17:40.805 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T12:17:40.805 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-21T12:17:40.805 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-21T12:17:40.805 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-21T12:17:40.808 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-21T12:17:40.808 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-21T12:17:40.808 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-21T12:17:40.808 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-21T12:17:40.808 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgmap": { 2024-09-21T12:17:40.808 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-21T12:17:40.808 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-21T12:17:40.808 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-21T12:17:40.808 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-21T12:17:40.808 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-21T12:17:40.808 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-21T12:17:40.808 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-21T12:17:40.808 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-21T12:17:40.808 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-21T12:17:40.809 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsmap": { 2024-09-21T12:17:40.809 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T12:17:40.809 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-21T12:17:40.809 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-21T12:17:40.809 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-21T12:17:40.809 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-21T12:17:40.809 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": false, 2024-09-21T12:17:40.809 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-21T12:17:40.809 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modules": [ 2024-09-21T12:17:40.809 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "iostat", 2024-09-21T12:17:40.809 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "nfs", 2024-09-21T12:17:40.809 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "restful" 2024-09-21T12:17:40.809 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-21T12:17:40.809 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-09-21T12:17:40.809 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-21T12:17:40.809 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "servicemap": { 2024-09-21T12:17:40.810 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T12:17:40.810 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modified": "2024-09-21T12:17:26.021949+0000", 2024-09-21T12:17:40.810 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-09-21T12:17:40.810 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-21T12:17:40.810 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-21T12:17:40.810 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-09-21T12:17:41.384 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:41 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/3643834289' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-21T12:17:41.385 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:41 smithi078 ceph-mon[23681]: mgrmap e3: smithi078.ntwyan(active, since 1.00547s) 2024-09-21T12:17:41.387 INFO:teuthology.orchestra.run.smithi078.stderr:mgr not available, waiting (3/15)... 2024-09-21T12:17:43.515 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:43 smithi078 ceph-mon[23681]: mgrmap e4: smithi078.ntwyan(active, since 2s) 2024-09-21T12:17:44.073 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-21T12:17:44.073 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-09-21T12:17:44.073 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsid": "6263deca-7813-11ef-baf6-efdc52797490", 2024-09-21T12:17:44.073 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "health": { 2024-09-21T12:17:44.074 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-21T12:17:44.074 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "checks": {}, 2024-09-21T12:17:44.074 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mutes": [] 2024-09-21T12:17:44.074 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-21T12:17:44.074 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-21T12:17:44.074 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum": [ 2024-09-21T12:17:44.074 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 0 2024-09-21T12:17:44.074 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-21T12:17:44.074 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-21T12:17:44.074 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "smithi078" 2024-09-21T12:17:44.074 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-21T12:17:44.074 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "quorum_age": 12, 2024-09-21T12:17:44.074 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "monmap": { 2024-09-21T12:17:44.075 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T12:17:44.075 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-21T12:17:44.075 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-21T12:17:44.075 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-21T12:17:44.075 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osdmap": { 2024-09-21T12:17:44.075 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T12:17:44.075 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-21T12:17:44.075 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-21T12:17:44.075 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-21T12:17:44.075 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-21T12:17:44.078 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-21T12:17:44.078 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-21T12:17:44.078 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-21T12:17:44.078 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgmap": { 2024-09-21T12:17:44.078 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-21T12:17:44.079 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-21T12:17:44.079 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-21T12:17:44.079 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-21T12:17:44.079 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-21T12:17:44.079 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-21T12:17:44.079 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-21T12:17:44.079 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-21T12:17:44.079 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-21T12:17:44.079 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "fsmap": { 2024-09-21T12:17:44.079 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T12:17:44.079 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-21T12:17:44.079 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-21T12:17:44.079 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-21T12:17:44.079 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-21T12:17:44.079 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": true, 2024-09-21T12:17:44.080 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-21T12:17:44.080 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modules": [ 2024-09-21T12:17:44.080 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "iostat", 2024-09-21T12:17:44.080 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "nfs", 2024-09-21T12:17:44.080 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "restful" 2024-09-21T12:17:44.080 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ], 2024-09-21T12:17:44.080 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-09-21T12:17:44.080 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-21T12:17:44.080 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "servicemap": { 2024-09-21T12:17:44.080 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-21T12:17:44.080 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "modified": "2024-09-21T12:17:26.021949+0000", 2024-09-21T12:17:44.080 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "services": {} 2024-09-21T12:17:44.080 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: }, 2024-09-21T12:17:44.080 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-21T12:17:44.080 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-09-21T12:17:44.538 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:44 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/2405232921' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-21T12:17:44.676 INFO:teuthology.orchestra.run.smithi078.stderr:mgr is available 2024-09-21T12:17:45.305 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-21T12:17:45.305 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [global] 2024-09-21T12:17:45.305 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: fsid = 6263deca-7813-11ef-baf6-efdc52797490 2024-09-21T12:17:45.305 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-21T12:17:45.305 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-21T12:17:45.305 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-21T12:17:45.305 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-21T12:17:45.305 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-21T12:17:45.306 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [mgr] 2024-09-21T12:17:45.306 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-21T12:17:45.306 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 2024-09-21T12:17:45.306 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: [osd] 2024-09-21T12:17:45.306 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-21T12:17:45.306 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-21T12:17:45.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:45 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/712014891' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-21T12:17:45.874 INFO:teuthology.orchestra.run.smithi078.stderr:Enabling cephadm module... 2024-09-21T12:17:46.911 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:46 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/2753256482' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-21T12:17:47.738 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:47 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/2753256482' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-21T12:17:47.739 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:47 smithi078 ceph-mon[23681]: mgrmap e5: smithi078.ntwyan(active, since 6s) 2024-09-21T12:17:47.902 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-09-21T12:17:47.902 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 5, 2024-09-21T12:17:47.903 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": true, 2024-09-21T12:17:47.903 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "active_name": "smithi078.ntwyan", 2024-09-21T12:17:47.903 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-21T12:17:47.903 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-09-21T12:17:48.520 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for the mgr to restart... 2024-09-21T12:17:48.520 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mgr epoch 5... 2024-09-21T12:17:48.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:48 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/2792064213' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-21T12:17:53.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:53 smithi078 ceph-mon[23681]: Active manager daemon smithi078.ntwyan restarted 2024-09-21T12:17:53.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:53 smithi078 ceph-mon[23681]: Activating manager daemon smithi078.ntwyan 2024-09-21T12:17:53.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:53 smithi078 ceph-mon[23681]: osdmap e2: 0 total, 0 up, 0 in 2024-09-21T12:17:54.053 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-09-21T12:17:54.053 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-09-21T12:17:54.053 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "initialized": true 2024-09-21T12:17:54.053 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-09-21T12:17:54.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:54 smithi078 ceph-mon[23681]: mgrmap e6: smithi078.ntwyan(active, starting, since 0.361078s) 2024-09-21T12:17:54.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:54 smithi078 ceph-mon[23681]: from='mgr.14120 172.21.15.78:0/1718043946' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-21T12:17:54.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:54 smithi078 ceph-mon[23681]: from='mgr.14120 172.21.15.78:0/1718043946' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mgr metadata", "who": "smithi078.ntwyan", "id": "smithi078.ntwyan"}]: dispatch 2024-09-21T12:17:54.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:54 smithi078 ceph-mon[23681]: from='mgr.14120 172.21.15.78:0/1718043946' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T12:17:54.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:54 smithi078 ceph-mon[23681]: from='mgr.14120 172.21.15.78:0/1718043946' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T12:17:54.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:54 smithi078 ceph-mon[23681]: from='mgr.14120 172.21.15.78:0/1718043946' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T12:17:54.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:54 smithi078 ceph-mon[23681]: Manager daemon smithi078.ntwyan is now available 2024-09-21T12:17:54.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:54 smithi078 ceph-mon[23681]: from='mgr.14120 172.21.15.78:0/1718043946' entity='mgr.smithi078.ntwyan' 2024-09-21T12:17:54.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:54 smithi078 ceph-mon[23681]: from='mgr.14120 172.21.15.78:0/1718043946' entity='mgr.smithi078.ntwyan' 2024-09-21T12:17:54.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:54 smithi078 ceph-mon[23681]: from='mgr.14120 172.21.15.78:0/1718043946' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:17:54.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:54 smithi078 ceph-mon[23681]: from='mgr.14120 172.21.15.78:0/1718043946' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:17:54.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:54 smithi078 ceph-mon[23681]: from='mgr.14120 172.21.15.78:0/1718043946' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:17:54.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:54 smithi078 ceph-mon[23681]: from='mgr.14120 172.21.15.78:0/1718043946' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/mirror_snapshot_schedule"}]: dispatch 2024-09-21T12:17:54.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:54 smithi078 ceph-mon[23681]: from='mgr.14120 172.21.15.78:0/1718043946' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/trash_purge_schedule"}]: dispatch 2024-09-21T12:17:54.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:54 smithi078 ceph-mon[23681]: [21/Sep/2024:12:17:53] ENGINE Bus STARTING 2024-09-21T12:17:54.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:54 smithi078 ceph-mon[23681]: [21/Sep/2024:12:17:53] ENGINE Serving on https://172.21.15.78:7150 2024-09-21T12:17:54.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:54 smithi078 ceph-mon[23681]: [21/Sep/2024:12:17:53] ENGINE Bus STARTED 2024-09-21T12:17:54.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:54 smithi078 ceph-mon[23681]: from='mgr.14120 172.21.15.78:0/1718043946' entity='mgr.smithi078.ntwyan' 2024-09-21T12:17:54.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:54 smithi078 ceph-mon[23681]: from='mgr.14120 172.21.15.78:0/1718043946' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:17:54.569 INFO:teuthology.orchestra.run.smithi078.stderr:mgr epoch 5 is available 2024-09-21T12:17:54.569 INFO:teuthology.orchestra.run.smithi078.stderr:Setting orchestrator backend to cephadm... 2024-09-21T12:17:55.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:55 smithi078 ceph-mon[23681]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-21T12:17:55.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:55 smithi078 ceph-mon[23681]: mgrmap e7: smithi078.ntwyan(active, since 1.36764s) 2024-09-21T12:17:55.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:55 smithi078 ceph-mon[23681]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-21T12:17:56.132 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:56 smithi078 ceph-mon[23681]: mgrmap e8: smithi078.ntwyan(active, since 2s) 2024-09-21T12:17:56.132 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:56 smithi078 ceph-mon[23681]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:17:56.133 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:56 smithi078 ceph-mon[23681]: from='mgr.14120 172.21.15.78:0/1718043946' entity='mgr.smithi078.ntwyan' 2024-09-21T12:17:56.133 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:56 smithi078 ceph-mon[23681]: from='mgr.14120 172.21.15.78:0/1718043946' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:17:56.408 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: value unchanged 2024-09-21T12:17:57.005 INFO:teuthology.orchestra.run.smithi078.stderr:Generating ssh key... 2024-09-21T12:17:57.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:57 smithi078 ceph-mon[23681]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:17:58.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:58 smithi078 ceph-mon[23681]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:17:58.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:58 smithi078 ceph-mon[23681]: Generating ssh key... 2024-09-21T12:17:58.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:58 smithi078 ceph-mon[23681]: from='mgr.14120 172.21.15.78:0/1718043946' entity='mgr.smithi078.ntwyan' 2024-09-21T12:17:58.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:58 smithi078 ceph-mon[23681]: from='mgr.14120 172.21.15.78:0/1718043946' entity='mgr.smithi078.ntwyan' 2024-09-21T12:17:58.949 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDDV5NMeeZC+OtJSCiRh5xjMBrU54SatKY1PLrRUKt09/fAW4AQGqmQMDNtIr5htEwYlDsXtxsEer3ZO1Ok4BX1TAZm2b/yzjiVAo5OQ9OBgP+XBfC7cb9Tquq3HhO/oPXcPG9ScipKDYhh4M5aufaj+kB7CVArK6soGY8pmQUhWBvgwq3DzRBkXjamogAL+E9JgS1KweLPQUmS9AaCtAtGHBt9EMbLUbt6U8cABmLhdYeDf6g4pynr+HOJnpuFkJjnBPph7p+qLiBjZKzVKCdFbUMK1QXraskn5rAO0j8p0D4SPtu09g3Y/XLbONCs3IaPyec5v76DykXhzrG4oV0HYEf7DdSKYcnmLFZah0ZSoxWVCvBZdDC5y7XL6RhsqMhVHaXASJd95OMHeqmCAh1GnJftlz/A1eT1pizMJ1u+nToZc4ynG+SDSN9IwRyRwAeMu/KSfbCTfZn9XX6mH5n4OV1+5aIfwim3QmPV5KnOcGlj3A2r3R68lFcZ4wNjtJU= ceph-6263deca-7813-11ef-baf6-efdc52797490 2024-09-21T12:17:59.543 INFO:teuthology.orchestra.run.smithi078.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-21T12:17:59.544 INFO:teuthology.orchestra.run.smithi078.stderr:Adding key to root@localhost authorized_keys... 2024-09-21T12:17:59.549 INFO:teuthology.orchestra.run.smithi078.stderr:Adding host smithi078... 2024-09-21T12:17:59.779 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:17:59 smithi078 ceph-mon[23681]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:18:00.798 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:00 smithi078 ceph-mon[23681]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi078", "addr": "172.21.15.78", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:18:00.798 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:00 smithi078 ceph-mon[23681]: Deploying cephadm binary to smithi078 2024-09-21T12:18:01.028 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Added host 'smithi078' with addr '172.21.15.78' 2024-09-21T12:18:01.609 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying mon service with default placement... 2024-09-21T12:18:02.256 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:02 smithi078 ceph-mon[23681]: from='mgr.14120 172.21.15.78:0/1718043946' entity='mgr.smithi078.ntwyan' 2024-09-21T12:18:02.256 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:02 smithi078 ceph-mon[23681]: Added host smithi078 2024-09-21T12:18:02.256 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:02 smithi078 ceph-mon[23681]: from='mgr.14120 172.21.15.78:0/1718043946' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:18:02.256 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled mon update... 2024-09-21T12:18:02.960 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying mgr service with default placement... 2024-09-21T12:18:03.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:03 smithi078 ceph-mon[23681]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:18:03.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:03 smithi078 ceph-mon[23681]: Saving service mon spec with placement count:5 2024-09-21T12:18:03.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:03 smithi078 ceph-mon[23681]: from='mgr.14120 172.21.15.78:0/1718043946' entity='mgr.smithi078.ntwyan' 2024-09-21T12:18:03.706 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-09-21T12:18:04.576 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying crash service with default placement... 2024-09-21T12:18:05.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:04 smithi078 ceph-mon[23681]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:18:05.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:04 smithi078 ceph-mon[23681]: Saving service mgr spec with placement count:2 2024-09-21T12:18:05.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:04 smithi078 ceph-mon[23681]: from='mgr.14120 172.21.15.78:0/1718043946' entity='mgr.smithi078.ntwyan' 2024-09-21T12:18:05.163 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled crash update... 2024-09-21T12:18:05.902 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying prometheus service with default placement... 2024-09-21T12:18:06.419 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:06 smithi078 ceph-mon[23681]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:18:06.419 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:06 smithi078 ceph-mon[23681]: Saving service crash spec with placement * 2024-09-21T12:18:06.419 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:06 smithi078 ceph-mon[23681]: from='mgr.14120 172.21.15.78:0/1718043946' entity='mgr.smithi078.ntwyan' 2024-09-21T12:18:06.419 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:06 smithi078 ceph-mon[23681]: from='mgr.14120 172.21.15.78:0/1718043946' entity='mgr.smithi078.ntwyan' 2024-09-21T12:18:06.752 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-09-21T12:18:07.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:07 smithi078 ceph-mon[23681]: from='mgr.14120 172.21.15.78:0/1718043946' entity='mgr.smithi078.ntwyan' 2024-09-21T12:18:07.615 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:07 smithi078 ceph-mon[23681]: from='mgr.14120 172.21.15.78:0/1718043946' entity='mgr.smithi078.ntwyan' 2024-09-21T12:18:07.995 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying grafana service with default placement... 2024-09-21T12:18:08.584 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:08 smithi078 ceph-mon[23681]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:18:08.585 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:08 smithi078 ceph-mon[23681]: Saving service prometheus spec with placement count:1 2024-09-21T12:18:08.585 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:08 smithi078 ceph-mon[23681]: from='mgr.14120 172.21.15.78:0/1718043946' entity='mgr.smithi078.ntwyan' 2024-09-21T12:18:08.585 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:08 smithi078 ceph-mon[23681]: from='mgr.14120 172.21.15.78:0/1718043946' entity='mgr.smithi078.ntwyan' 2024-09-21T12:18:08.636 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-09-21T12:18:09.664 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying node-exporter service with default placement... 2024-09-21T12:18:09.884 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:09 smithi078 ceph-mon[23681]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:18:09.884 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:09 smithi078 ceph-mon[23681]: Saving service grafana spec with placement count:1 2024-09-21T12:18:09.884 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:09 smithi078 ceph-mon[23681]: from='mgr.14120 172.21.15.78:0/1718043946' entity='mgr.smithi078.ntwyan' 2024-09-21T12:18:10.268 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-09-21T12:18:11.113 INFO:teuthology.orchestra.run.smithi078.stderr:Deploying alertmanager service with default placement... 2024-09-21T12:18:11.369 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:11 smithi078 ceph-mon[23681]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:18:11.369 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:11 smithi078 ceph-mon[23681]: Saving service node-exporter spec with placement * 2024-09-21T12:18:11.369 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:11 smithi078 ceph-mon[23681]: from='mgr.14120 172.21.15.78:0/1718043946' entity='mgr.smithi078.ntwyan' 2024-09-21T12:18:11.792 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-09-21T12:18:12.920 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:12 smithi078 ceph-mon[23681]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:18:12.920 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:12 smithi078 ceph-mon[23681]: Saving service alertmanager spec with placement count:1 2024-09-21T12:18:12.920 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:12 smithi078 ceph-mon[23681]: from='mgr.14120 172.21.15.78:0/1718043946' entity='mgr.smithi078.ntwyan' 2024-09-21T12:18:14.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:14 smithi078 ceph-mon[23681]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:18:14.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:14 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/1917040174' entity='client.admin' 2024-09-21T12:18:15.085 INFO:teuthology.orchestra.run.smithi078.stderr:Enabling the dashboard module... 2024-09-21T12:18:15.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:15 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/4082234365' entity='client.admin' 2024-09-21T12:18:16.708 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:16 smithi078 ceph-mon[23681]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:18:16.708 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:16 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/3407494780' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-21T12:18:17.642 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:17 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/3407494780' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-21T12:18:17.642 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:17 smithi078 ceph-mon[23681]: mgrmap e9: smithi078.ntwyan(active, since 23s) 2024-09-21T12:18:17.819 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-09-21T12:18:17.819 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "epoch": 9, 2024-09-21T12:18:17.819 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "available": true, 2024-09-21T12:18:17.819 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "active_name": "smithi078.ntwyan", 2024-09-21T12:18:17.819 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-21T12:18:17.819 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-09-21T12:18:18.457 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for the mgr to restart... 2024-09-21T12:18:18.457 INFO:teuthology.orchestra.run.smithi078.stderr:Waiting for mgr epoch 9... 2024-09-21T12:18:18.716 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:18 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/3217819353' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-21T12:18:23.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:23 smithi078 ceph-mon[23681]: Active manager daemon smithi078.ntwyan restarted 2024-09-21T12:18:23.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:23 smithi078 ceph-mon[23681]: Activating manager daemon smithi078.ntwyan 2024-09-21T12:18:23.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:23 smithi078 ceph-mon[23681]: osdmap e3: 0 total, 0 up, 0 in 2024-09-21T12:18:24.061 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: { 2024-09-21T12:18:24.061 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-09-21T12:18:24.061 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: "initialized": true 2024-09-21T12:18:24.061 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: } 2024-09-21T12:18:24.200 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:24 smithi078 ceph-mon[23681]: mgrmap e10: smithi078.ntwyan(active, starting, since 0.460482s) 2024-09-21T12:18:24.200 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:24 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-21T12:18:24.200 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:24 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mgr metadata", "who": "smithi078.ntwyan", "id": "smithi078.ntwyan"}]: dispatch 2024-09-21T12:18:24.200 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:24 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T12:18:24.200 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:24 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T12:18:24.200 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:24 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T12:18:24.200 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:24 smithi078 ceph-mon[23681]: Manager daemon smithi078.ntwyan is now available 2024-09-21T12:18:24.200 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:24 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:18:24.200 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:24 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:18:24.201 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:24 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/mirror_snapshot_schedule"}]: dispatch 2024-09-21T12:18:24.201 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:24 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/trash_purge_schedule"}]: dispatch 2024-09-21T12:18:24.201 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:24 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' 2024-09-21T12:18:24.784 INFO:teuthology.orchestra.run.smithi078.stderr:mgr epoch 9 is available 2024-09-21T12:18:24.784 INFO:teuthology.orchestra.run.smithi078.stderr:Generating a dashboard self-signed certificate... 2024-09-21T12:18:25.053 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:25 smithi078 ceph-mon[23681]: [21/Sep/2024:12:18:23] ENGINE Bus STARTING 2024-09-21T12:18:25.053 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:25 smithi078 ceph-mon[23681]: [21/Sep/2024:12:18:23] ENGINE Serving on https://172.21.15.78:7150 2024-09-21T12:18:25.053 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:25 smithi078 ceph-mon[23681]: [21/Sep/2024:12:18:23] ENGINE Bus STARTED 2024-09-21T12:18:25.053 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:25 smithi078 ceph-mon[23681]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-21T12:18:25.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:25 smithi078 ceph-mon[23681]: mgrmap e11: smithi078.ntwyan(active, since 1.4671s) 2024-09-21T12:18:25.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:25 smithi078 ceph-mon[23681]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-21T12:18:25.564 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: Self-signed certificate created 2024-09-21T12:18:26.216 INFO:teuthology.orchestra.run.smithi078.stderr:Creating initial admin user... 2024-09-21T12:18:26.340 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:26 smithi078 ceph-mon[23681]: mgrmap e12: smithi078.ntwyan(active, since 2s) 2024-09-21T12:18:26.340 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:26 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' 2024-09-21T12:18:26.340 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:26 smithi078 ceph-mon[23681]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:18:26.340 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:26 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' 2024-09-21T12:18:26.340 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:26 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' 2024-09-21T12:18:26.340 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:26 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' 2024-09-21T12:18:27.106 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$DODu5melMpijOgCFwxBH5.e5go5EY5mV7V2bvw/MPa5RvhLrYF/tO", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726921107, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-21T12:18:27.919 INFO:teuthology.orchestra.run.smithi078.stderr:Fetching dashboard port number... 2024-09-21T12:18:28.171 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:28 smithi078 ceph-mon[23681]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:18:28.171 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:28 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' 2024-09-21T12:18:28.565 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: 8443 2024-09-21T12:18:29.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:29 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/1967951309' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-21T12:18:29.421 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: disabled 2024-09-21T12:18:29.428 INFO:teuthology.orchestra.run.smithi078.stderr:systemctl: inactive 2024-09-21T12:18:29.429 INFO:teuthology.orchestra.run.smithi078.stderr:firewalld.service is not enabled 2024-09-21T12:18:29.429 INFO:teuthology.orchestra.run.smithi078.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-21T12:18:29.431 INFO:teuthology.orchestra.run.smithi078.stderr:Ceph Dashboard is now available at: 2024-09-21T12:18:29.431 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-21T12:18:29.431 INFO:teuthology.orchestra.run.smithi078.stderr: URL: https://smithi078.front.sepia.ceph.com:8443/ 2024-09-21T12:18:29.431 INFO:teuthology.orchestra.run.smithi078.stderr: User: admin 2024-09-21T12:18:29.431 INFO:teuthology.orchestra.run.smithi078.stderr: Password: iitr3nuwvn 2024-09-21T12:18:29.431 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-21T12:18:29.431 INFO:teuthology.orchestra.run.smithi078.stderr:Enabling autotune for osd_memory_target 2024-09-21T12:18:30.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:30 smithi078 ceph-mon[23681]: mgrmap e13: smithi078.ntwyan(active, since 6s) 2024-09-21T12:18:31.492 INFO:teuthology.orchestra.run.smithi078.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-09-21T12:18:32.092 INFO:teuthology.orchestra.run.smithi078.stderr:You can access the Ceph CLI with: 2024-09-21T12:18:32.092 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-21T12:18:32.092 INFO:teuthology.orchestra.run.smithi078.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 6263deca-7813-11ef-baf6-efdc52797490 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-21T12:18:32.092 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-21T12:18:32.092 INFO:teuthology.orchestra.run.smithi078.stderr:Please consider enabling telemetry to help improve Ceph: 2024-09-21T12:18:32.092 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-21T12:18:32.092 INFO:teuthology.orchestra.run.smithi078.stderr: ceph telemetry on 2024-09-21T12:18:32.092 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-21T12:18:32.092 INFO:teuthology.orchestra.run.smithi078.stderr:For more information see: 2024-09-21T12:18:32.093 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-21T12:18:32.093 INFO:teuthology.orchestra.run.smithi078.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-21T12:18:32.093 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-21T12:18:32.093 INFO:teuthology.orchestra.run.smithi078.stderr:Bootstrap complete. 2024-09-21T12:18:32.129 INFO:tasks.cephadm:Fetching config... 2024-09-21T12:18:32.130 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T12:18:32.130 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-21T12:18:32.145 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-21T12:18:32.145 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T12:18:32.145 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-21T12:18:32.200 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-21T12:18:32.200 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T12:18:32.200 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/mon.smithi078/keyring of=/dev/stdout 2024-09-21T12:18:32.266 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-21T12:18:32.266 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T12:18:32.266 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-21T12:18:32.321 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-21T12:18:32.322 DEBUG:teuthology.orchestra.run.smithi078:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDDV5NMeeZC+OtJSCiRh5xjMBrU54SatKY1PLrRUKt09/fAW4AQGqmQMDNtIr5htEwYlDsXtxsEer3ZO1Ok4BX1TAZm2b/yzjiVAo5OQ9OBgP+XBfC7cb9Tquq3HhO/oPXcPG9ScipKDYhh4M5aufaj+kB7CVArK6soGY8pmQUhWBvgwq3DzRBkXjamogAL+E9JgS1KweLPQUmS9AaCtAtGHBt9EMbLUbt6U8cABmLhdYeDf6g4pynr+HOJnpuFkJjnBPph7p+qLiBjZKzVKCdFbUMK1QXraskn5rAO0j8p0D4SPtu09g3Y/XLbONCs3IaPyec5v76DykXhzrG4oV0HYEf7DdSKYcnmLFZah0ZSoxWVCvBZdDC5y7XL6RhsqMhVHaXASJd95OMHeqmCAh1GnJftlz/A1eT1pizMJ1u+nToZc4ynG+SDSN9IwRyRwAeMu/KSfbCTfZn9XX6mH5n4OV1+5aIfwim3QmPV5KnOcGlj3A2r3R68lFcZ4wNjtJU= ceph-6263deca-7813-11ef-baf6-efdc52797490' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-21T12:18:32.405 INFO:teuthology.orchestra.run.smithi078.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDDV5NMeeZC+OtJSCiRh5xjMBrU54SatKY1PLrRUKt09/fAW4AQGqmQMDNtIr5htEwYlDsXtxsEer3ZO1Ok4BX1TAZm2b/yzjiVAo5OQ9OBgP+XBfC7cb9Tquq3HhO/oPXcPG9ScipKDYhh4M5aufaj+kB7CVArK6soGY8pmQUhWBvgwq3DzRBkXjamogAL+E9JgS1KweLPQUmS9AaCtAtGHBt9EMbLUbt6U8cABmLhdYeDf6g4pynr+HOJnpuFkJjnBPph7p+qLiBjZKzVKCdFbUMK1QXraskn5rAO0j8p0D4SPtu09g3Y/XLbONCs3IaPyec5v76DykXhzrG4oV0HYEf7DdSKYcnmLFZah0ZSoxWVCvBZdDC5y7XL6RhsqMhVHaXASJd95OMHeqmCAh1GnJftlz/A1eT1pizMJ1u+nToZc4ynG+SDSN9IwRyRwAeMu/KSfbCTfZn9XX6mH5n4OV1+5aIfwim3QmPV5KnOcGlj3A2r3R68lFcZ4wNjtJU= ceph-6263deca-7813-11ef-baf6-efdc52797490 2024-09-21T12:18:32.416 DEBUG:teuthology.orchestra.run.smithi148:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDDV5NMeeZC+OtJSCiRh5xjMBrU54SatKY1PLrRUKt09/fAW4AQGqmQMDNtIr5htEwYlDsXtxsEer3ZO1Ok4BX1TAZm2b/yzjiVAo5OQ9OBgP+XBfC7cb9Tquq3HhO/oPXcPG9ScipKDYhh4M5aufaj+kB7CVArK6soGY8pmQUhWBvgwq3DzRBkXjamogAL+E9JgS1KweLPQUmS9AaCtAtGHBt9EMbLUbt6U8cABmLhdYeDf6g4pynr+HOJnpuFkJjnBPph7p+qLiBjZKzVKCdFbUMK1QXraskn5rAO0j8p0D4SPtu09g3Y/XLbONCs3IaPyec5v76DykXhzrG4oV0HYEf7DdSKYcnmLFZah0ZSoxWVCvBZdDC5y7XL6RhsqMhVHaXASJd95OMHeqmCAh1GnJftlz/A1eT1pizMJ1u+nToZc4ynG+SDSN9IwRyRwAeMu/KSfbCTfZn9XX6mH5n4OV1+5aIfwim3QmPV5KnOcGlj3A2r3R68lFcZ4wNjtJU= ceph-6263deca-7813-11ef-baf6-efdc52797490' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-21T12:18:32.552 INFO:teuthology.orchestra.run.smithi148.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDDV5NMeeZC+OtJSCiRh5xjMBrU54SatKY1PLrRUKt09/fAW4AQGqmQMDNtIr5htEwYlDsXtxsEer3ZO1Ok4BX1TAZm2b/yzjiVAo5OQ9OBgP+XBfC7cb9Tquq3HhO/oPXcPG9ScipKDYhh4M5aufaj+kB7CVArK6soGY8pmQUhWBvgwq3DzRBkXjamogAL+E9JgS1KweLPQUmS9AaCtAtGHBt9EMbLUbt6U8cABmLhdYeDf6g4pynr+HOJnpuFkJjnBPph7p+qLiBjZKzVKCdFbUMK1QXraskn5rAO0j8p0D4SPtu09g3Y/XLbONCs3IaPyec5v76DykXhzrG4oV0HYEf7DdSKYcnmLFZah0ZSoxWVCvBZdDC5y7XL6RhsqMhVHaXASJd95OMHeqmCAh1GnJftlz/A1eT1pizMJ1u+nToZc4ynG+SDSN9IwRyRwAeMu/KSfbCTfZn9XX6mH5n4OV1+5aIfwim3QmPV5KnOcGlj3A2r3R68lFcZ4wNjtJU= ceph-6263deca-7813-11ef-baf6-efdc52797490 2024-09-21T12:18:32.564 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-21T12:18:32.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:32 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/2443582310' entity='client.admin' 2024-09-21T12:18:36.351 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-21T12:18:36.351 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-21T12:18:36.933 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:36 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/2320718681' entity='client.admin' 2024-09-21T12:18:38.533 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:38 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' 2024-09-21T12:18:38.534 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:38 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd/host:smithi078", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:18:38.534 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:38 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' 2024-09-21T12:18:38.534 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:38 smithi078 ceph-mon[23681]: Deploying daemon alertmanager.smithi078 on smithi078 2024-09-21T12:18:39.928 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi148 2024-09-21T12:18:39.929 DEBUG:teuthology.orchestra.run.smithi148:> set -ex 2024-09-21T12:18:39.929 DEBUG:teuthology.orchestra.run.smithi148:> dd of=/etc/ceph/ceph.conf 2024-09-21T12:18:39.947 DEBUG:teuthology.orchestra.run.smithi148:> set -ex 2024-09-21T12:18:39.947 DEBUG:teuthology.orchestra.run.smithi148:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-21T12:18:40.005 INFO:tasks.cephadm:Adding host smithi148 to orchestrator... 2024-09-21T12:18:40.005 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph orch host add smithi148 2024-09-21T12:18:40.027 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:39 smithi078 ceph-mon[23681]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:18:40.027 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:39 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' 2024-09-21T12:18:44.131 INFO:teuthology.orchestra.run.smithi078.stdout:Added host 'smithi148' with addr '172.21.15.148' 2024-09-21T12:18:44.461 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:44 smithi078 ceph-mon[23681]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:18:44.462 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:44 smithi078 ceph-mon[23681]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi148", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:18:45.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:45 smithi078 ceph-mon[23681]: Deploying cephadm binary to smithi148 2024-09-21T12:18:45.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:45 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' 2024-09-21T12:18:45.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:45 smithi078 ceph-mon[23681]: Added host smithi148 2024-09-21T12:18:45.880 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph orch host ls --format=json 2024-09-21T12:18:46.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:46 smithi078 ceph-mon[23681]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:18:48.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:48 smithi078 ceph-mon[23681]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:18:48.367 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T12:18:48.367 INFO:teuthology.orchestra.run.smithi078.stdout:[{"addr": "172.21.15.78", "hostname": "smithi078", "labels": [], "status": ""}, {"addr": "172.21.15.148", "hostname": "smithi148", "labels": [], "status": ""}] 2024-09-21T12:18:49.190 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:49 smithi078 ceph-mon[23681]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T12:18:49.522 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-21T12:18:49.522 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph osd crush tunables default 2024-09-21T12:18:50.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:50 smithi078 ceph-mon[23681]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:18:50.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:50 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' 2024-09-21T12:18:50.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:50 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-21T12:18:50.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:50 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-21T12:18:50.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:50 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:18:50.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:50 smithi078 ceph-mon[23681]: Deploying daemon crash.smithi078 on smithi078 2024-09-21T12:18:52.238 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:52 smithi078 ceph-mon[23681]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:18:53.078 INFO:teuthology.orchestra.run.smithi078.stderr:adjusted tunables profile to default 2024-09-21T12:18:53.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:53 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/3791962968' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-21T12:18:53.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:53 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' 2024-09-21T12:18:53.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:53 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' 2024-09-21T12:18:53.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:53 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' 2024-09-21T12:18:53.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:53 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T12:18:53.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:53 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' 2024-09-21T12:18:53.734 INFO:tasks.cephadm:Adding mon.smithi078 on smithi078 2024-09-21T12:18:53.734 INFO:tasks.cephadm:Adding mon.smithi148 on smithi148 2024-09-21T12:18:53.735 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph orch apply mon '2;smithi078:172.21.15.78=smithi078;smithi148:172.21.15.148=smithi148' 2024-09-21T12:18:54.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:54 smithi078 ceph-mon[23681]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T12:18:54.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:54 smithi078 ceph-mon[23681]: Deploying daemon grafana.smithi078 on smithi078 2024-09-21T12:18:54.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:54 smithi078 ceph-mon[23681]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:18:54.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:54 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/3791962968' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-21T12:18:54.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:54 smithi078 ceph-mon[23681]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T12:18:54.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:54 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' 2024-09-21T12:18:55.553 INFO:teuthology.orchestra.run.smithi148.stdout:Scheduled mon update... 2024-09-21T12:18:56.132 DEBUG:teuthology.orchestra.run.smithi148:mon.smithi148> sudo journalctl -f -n 0 -u ceph-6263deca-7813-11ef-baf6-efdc52797490@mon.smithi148.service 2024-09-21T12:18:56.135 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T12:18:56.135 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T12:18:56.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:56 smithi078 ceph-mon[23681]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:18:56.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:56 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' 2024-09-21T12:18:57.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:57 smithi078 ceph-mon[23681]: from='client.14194 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi078:172.21.15.78=smithi078;smithi148:172.21.15.148=smithi148", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:18:57.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:57 smithi078 ceph-mon[23681]: Saving service mon spec with placement smithi078:172.21.15.78=smithi078;smithi148:172.21.15.148=smithi148;count:2 2024-09-21T12:18:57.936 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-09-21T12:18:57.936 INFO:teuthology.orchestra.run.smithi148.stdout:{"epoch":1,"fsid":"6263deca-7813-11ef-baf6-efdc52797490","modified":"2024-09-21T12:17:23.120662Z","created":"2024-09-21T12:17:23.120662Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T12:18:57.938 INFO:teuthology.orchestra.run.smithi148.stderr:dumped monmap epoch 1 2024-09-21T12:18:58.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:58 smithi078 ceph-mon[23681]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:18:58.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:18:58 smithi078 ceph-mon[23681]: from='client.? 172.21.15.148:0/1747267838' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T12:18:59.525 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T12:18:59.526 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T12:19:00.483 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:00 smithi078 ceph-mon[23681]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:01.513 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-09-21T12:19:01.513 INFO:teuthology.orchestra.run.smithi148.stdout:{"epoch":1,"fsid":"6263deca-7813-11ef-baf6-efdc52797490","modified":"2024-09-21T12:17:23.120662Z","created":"2024-09-21T12:17:23.120662Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T12:19:01.515 INFO:teuthology.orchestra.run.smithi148.stderr:dumped monmap epoch 1 2024-09-21T12:19:02.099 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:02 smithi078 ceph-mon[23681]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:02.099 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:02 smithi078 ceph-mon[23681]: from='client.? 172.21.15.148:0/3816282307' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T12:19:03.063 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T12:19:03.064 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T12:19:04.206 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:04 smithi078 ceph-mon[23681]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:04.206 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:04 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:04.812 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-09-21T12:19:04.812 INFO:teuthology.orchestra.run.smithi148.stdout:{"epoch":1,"fsid":"6263deca-7813-11ef-baf6-efdc52797490","modified":"2024-09-21T12:17:23.120662Z","created":"2024-09-21T12:17:23.120662Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T12:19:04.813 INFO:teuthology.orchestra.run.smithi148.stderr:dumped monmap epoch 1 2024-09-21T12:19:05.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:05 smithi078 ceph-mon[23681]: Deploying daemon node-exporter.smithi078 on smithi078 2024-09-21T12:19:05.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:05 smithi078 ceph-mon[23681]: from='client.? 172.21.15.148:0/434495684' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T12:19:06.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:06 smithi078 ceph-mon[23681]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:06.398 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T12:19:06.401 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T12:19:08.233 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-09-21T12:19:08.234 INFO:teuthology.orchestra.run.smithi148.stdout:{"epoch":1,"fsid":"6263deca-7813-11ef-baf6-efdc52797490","modified":"2024-09-21T12:17:23.120662Z","created":"2024-09-21T12:17:23.120662Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T12:19:08.235 INFO:teuthology.orchestra.run.smithi148.stderr:dumped monmap epoch 1 2024-09-21T12:19:08.296 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:07 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:08.296 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:07 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-21T12:19:08.296 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:07 smithi078 ceph-mon[23681]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:09.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:08 smithi078 ceph-mon[23681]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-21T12:19:09.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:08 smithi078 ceph-mon[23681]: mgrmap e14: smithi078.ntwyan(active, since 45s) 2024-09-21T12:19:09.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:08 smithi078 ceph-mon[23681]: from='client.? 172.21.15.148:0/379503574' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T12:19:09.842 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T12:19:09.842 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T12:19:11.688 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-09-21T12:19:11.688 INFO:teuthology.orchestra.run.smithi148.stdout:{"epoch":1,"fsid":"6263deca-7813-11ef-baf6-efdc52797490","modified":"2024-09-21T12:17:23.120662Z","created":"2024-09-21T12:17:23.120662Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T12:19:11.690 INFO:teuthology.orchestra.run.smithi148.stderr:dumped monmap epoch 1 2024-09-21T12:19:11.972 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:11 smithi078 ceph-mon[23681]: from='client.? 172.21.15.148:0/3021242459' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T12:19:13.264 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T12:19:13.264 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T12:19:15.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:14 smithi078 ceph-mon[23681]: Active manager daemon smithi078.ntwyan restarted 2024-09-21T12:19:15.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:14 smithi078 ceph-mon[23681]: Activating manager daemon smithi078.ntwyan 2024-09-21T12:19:15.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:14 smithi078 ceph-mon[23681]: osdmap e5: 0 total, 0 up, 0 in 2024-09-21T12:19:15.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:14 smithi078 ceph-mon[23681]: mgrmap e15: smithi078.ntwyan(active, starting, since 0.052298s) 2024-09-21T12:19:15.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-21T12:19:15.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mgr metadata", "who": "smithi078.ntwyan", "id": "smithi078.ntwyan"}]: dispatch 2024-09-21T12:19:15.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T12:19:15.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T12:19:15.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T12:19:15.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:14 smithi078 ceph-mon[23681]: Manager daemon smithi078.ntwyan is now available 2024-09-21T12:19:15.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:19:15.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:19:15.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:19:15.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:19:15.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:15.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/mirror_snapshot_schedule"}]: dispatch 2024-09-21T12:19:15.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/trash_purge_schedule"}]: dispatch 2024-09-21T12:19:15.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:15.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:15.202 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-09-21T12:19:15.202 INFO:teuthology.orchestra.run.smithi148.stdout:{"epoch":1,"fsid":"6263deca-7813-11ef-baf6-efdc52797490","modified":"2024-09-21T12:17:23.120662Z","created":"2024-09-21T12:17:23.120662Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T12:19:15.204 INFO:teuthology.orchestra.run.smithi148.stderr:dumped monmap epoch 1 2024-09-21T12:19:16.208 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:15 smithi078 ceph-mon[23681]: [21/Sep/2024:12:19:14] ENGINE Bus STARTING 2024-09-21T12:19:16.209 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:15 smithi078 ceph-mon[23681]: [21/Sep/2024:12:19:14] ENGINE Serving on https://172.21.15.78:7150 2024-09-21T12:19:16.209 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:15 smithi078 ceph-mon[23681]: [21/Sep/2024:12:19:14] ENGINE Bus STARTED 2024-09-21T12:19:16.209 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:15 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:16.209 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:15 smithi078 ceph-mon[23681]: mgrmap e16: smithi078.ntwyan(active, since 1.05682s) 2024-09-21T12:19:16.209 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:15 smithi078 ceph-mon[23681]: from='client.? 172.21.15.148:0/1896679403' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T12:19:16.953 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T12:19:16.953 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T12:19:17.214 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:16 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:17.214 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:16 smithi078 ceph-mon[23681]: mgrmap e17: smithi078.ntwyan(active, since 2s) 2024-09-21T12:19:18.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:18 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:18.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:18 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:18.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:18 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd/host:smithi078", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:19:18.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:18 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:19.224 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-09-21T12:19:19.224 INFO:teuthology.orchestra.run.smithi148.stdout:{"epoch":1,"fsid":"6263deca-7813-11ef-baf6-efdc52797490","modified":"2024-09-21T12:17:23.120662Z","created":"2024-09-21T12:17:23.120662Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T12:19:19.226 INFO:teuthology.orchestra.run.smithi148.stderr:dumped monmap epoch 1 2024-09-21T12:19:19.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:19 smithi078 ceph-mon[23681]: Updating smithi078:/etc/ceph/ceph.conf 2024-09-21T12:19:19.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:19 smithi078 ceph-mon[23681]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-09-21T12:19:19.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:19 smithi078 ceph-mon[23681]: from='client.? 172.21.15.148:0/4258519499' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T12:19:20.745 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T12:19:20.745 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T12:19:21.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:21 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:21.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:21 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:21.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:21 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd/host:smithi148", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:19:21.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:21 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:21.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:21 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi148", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-21T12:19:21.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:21 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi148", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-21T12:19:21.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:21 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:19:21.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:21 smithi078 ceph-mon[23681]: Deploying daemon crash.smithi148 on smithi148 2024-09-21T12:19:23.311 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-09-21T12:19:23.311 INFO:teuthology.orchestra.run.smithi148.stdout:{"epoch":1,"fsid":"6263deca-7813-11ef-baf6-efdc52797490","modified":"2024-09-21T12:17:23.120662Z","created":"2024-09-21T12:17:23.120662Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T12:19:23.313 INFO:teuthology.orchestra.run.smithi148.stderr:dumped monmap epoch 1 2024-09-21T12:19:23.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:23 smithi078 ceph-mon[23681]: from='client.? 172.21.15.148:0/2572010379' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T12:19:25.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:24 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:25.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:24 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi148.oyiqmx", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T12:19:25.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:24 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi148.oyiqmx", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-21T12:19:25.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:24 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T12:19:25.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:24 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:19:25.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:24 smithi078 ceph-mon[23681]: Deploying daemon mgr.smithi148.oyiqmx on smithi148 2024-09-21T12:19:25.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:24 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:25.117 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T12:19:25.117 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T12:19:27.215 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-09-21T12:19:27.215 INFO:teuthology.orchestra.run.smithi148.stdout:{"epoch":1,"fsid":"6263deca-7813-11ef-baf6-efdc52797490","modified":"2024-09-21T12:17:23.120662Z","created":"2024-09-21T12:17:23.120662Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T12:19:27.217 INFO:teuthology.orchestra.run.smithi148.stderr:dumped monmap epoch 1 2024-09-21T12:19:28.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:28 smithi078 ceph-mon[23681]: from='client.? 172.21.15.148:0/527304142' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T12:19:28.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:28 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:28.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:28 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T12:19:28.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:28 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:19:28.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:28 smithi078 ceph-mon[23681]: Deploying daemon mon.smithi148 on smithi148 2024-09-21T12:19:28.969 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T12:19:28.970 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T12:19:30.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:30 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:31.800 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-09-21T12:19:31.800 INFO:teuthology.orchestra.run.smithi148.stdout:{"epoch":1,"fsid":"6263deca-7813-11ef-baf6-efdc52797490","modified":"2024-09-21T12:17:23.120662Z","created":"2024-09-21T12:17:23.120662Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T12:19:31.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:31 smithi148 systemd[1]: Starting Ceph mon.smithi148 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:19:31.802 INFO:teuthology.orchestra.run.smithi148.stderr:dumped monmap epoch 1 2024-09-21T12:19:32.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:31 smithi078 ceph-mon[23681]: from='client.? 172.21.15.148:0/3700635901' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T12:19:32.599 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 podman[25776]: 2024-09-21 12:19:32.351966496 +0000 UTC m=+0.552706601 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-09-21T12:19:32.600 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 podman[25776]: 2024-09-21 12:19:32.477541183 +0000 UTC m=+0.678281275 container create fda961db230f48e5e21a74a156bb3cd9e8976d6341334f63421eb9aeb8f698fb (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi148, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, architecture=x86_64, release=754, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, ceph=True, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux ) 2024-09-21T12:19:33.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 podman[25776]: 2024-09-21 12:19:32.705248486 +0000 UTC m=+0.905988598 container init fda961db230f48e5e21a74a156bb3cd9e8976d6341334f63421eb9aeb8f698fb (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi148, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., version=8, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , vcs-type=git, io.openshift.tags=base centos centos-stream, ceph=True, io.openshift.expose-services=, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-09-21T12:19:33.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 podman[25776]: 2024-09-21 12:19:32.709487597 +0000 UTC m=+0.910227690 container start fda961db230f48e5e21a74a156bb3cd9e8976d6341334f63421eb9aeb8f698fb (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi148, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, architecture=x86_64, vcs-type=git, version=8, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, CEPH_POINT_RELEASE=-17.2.0, ceph=True, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , RELEASE=HEAD) 2024-09-21T12:19:33.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T12:19:33.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-21T12:19:33.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: pidfile_write: ignore empty --pid-file 2024-09-21T12:19:33.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: load: jerasure load: lrc 2024-09-21T12:19:33.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: RocksDB version: 6.15.5 2024-09-21T12:19:33.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-21T12:19:33.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Compile date Apr 18 2022 2024-09-21T12:19:33.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: DB SUMMARY 2024-09-21T12:19:33.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: DB Session ID: GNO3S8BDC11NGZCKD8TU 2024-09-21T12:19:33.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: CURRENT file: CURRENT 2024-09-21T12:19:33.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T12:19:33.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-21T12:19:33.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi148/store.db dir, Total Num: 0, files: 2024-09-21T12:19:33.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi148/store.db: 000004.log size: 511 ; 2024-09-21T12:19:33.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.error_if_exists: 0 2024-09-21T12:19:33.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.create_if_missing: 0 2024-09-21T12:19:33.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.paranoid_checks: 1 2024-09-21T12:19:33.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T12:19:33.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.env: 0x5629480c6860 2024-09-21T12:19:33.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.fs: Posix File System 2024-09-21T12:19:33.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.info_log: 0x562949769fa0 2024-09-21T12:19:33.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T12:19:33.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.statistics: (nil) 2024-09-21T12:19:33.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.use_fsync: 0 2024-09-21T12:19:33.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.max_log_file_size: 0 2024-09-21T12:19:33.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T12:19:33.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T12:19:33.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T12:19:33.054 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T12:19:33.054 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.allow_fallocate: 1 2024-09-21T12:19:33.054 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T12:19:33.054 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T12:19:33.054 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.use_direct_reads: 0 2024-09-21T12:19:33.054 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T12:19:33.054 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T12:19:33.054 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.db_log_dir: 2024-09-21T12:19:33.054 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi148/store.db 2024-09-21T12:19:33.054 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T12:19:33.054 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T12:19:33.054 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T12:19:33.054 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T12:19:33.055 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T12:19:33.055 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T12:19:33.056 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T12:19:33.056 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T12:19:33.056 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.write_buffer_manager: 0x56294985a330 2024-09-21T12:19:33.056 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T12:19:33.057 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-21T12:19:33.057 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T12:19:33.057 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T12:19:33.057 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T12:19:33.057 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T12:19:33.057 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T12:19:33.057 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T12:19:33.057 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T12:19:33.057 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.unordered_write: 0 2024-09-21T12:19:33.057 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T12:19:33.057 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T12:19:33.057 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T12:19:33.057 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T12:19:33.057 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.row_cache: None 2024-09-21T12:19:33.058 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.wal_filter: None 2024-09-21T12:19:33.058 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T12:19:33.058 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T12:19:33.058 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.preserve_deletes: 0 2024-09-21T12:19:33.058 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.two_write_queues: 0 2024-09-21T12:19:33.058 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T12:19:33.058 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.atomic_flush: 0 2024-09-21T12:19:33.058 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T12:19:33.058 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T12:19:33.058 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T12:19:33.058 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.log_readahead_size: 0 2024-09-21T12:19:33.058 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T12:19:33.058 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T12:19:33.058 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T12:19:33.058 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T12:19:33.059 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T12:19:33.059 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T12:19:33.059 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.max_background_jobs: 2 2024-09-21T12:19:33.059 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.max_background_compactions: -1 2024-09-21T12:19:33.059 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.max_subcompactions: 1 2024-09-21T12:19:33.059 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T12:19:33.059 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T12:19:33.059 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T12:19:33.059 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T12:19:33.059 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T12:19:33.059 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T12:19:33.059 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T12:19:33.059 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T12:19:33.059 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.max_open_files: -1 2024-09-21T12:19:33.060 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T12:19:33.060 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T12:19:33.060 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T12:19:33.060 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T12:19:33.060 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.max_background_flushes: -1 2024-09-21T12:19:33.060 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Compression algorithms supported: 2024-09-21T12:19:33.060 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T12:19:33.060 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: kZSTD supported: 0 2024-09-21T12:19:33.060 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: kXpressCompression supported: 0 2024-09-21T12:19:33.060 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T12:19:33.060 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: kLZ4Compression supported: 1 2024-09-21T12:19:33.060 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: kBZip2Compression supported: 0 2024-09-21T12:19:33.060 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: kZlibCompression supported: 1 2024-09-21T12:19:33.060 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: kSnappyCompression supported: 1 2024-09-21T12:19:33.061 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T12:19:33.061 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi148/store.db/MANIFEST-000003 2024-09-21T12:19:33.061 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-21T12:19:33.061 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T12:19:33.061 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.merge_operator: 2024-09-21T12:19:33.061 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.compaction_filter: None 2024-09-21T12:19:33.061 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T12:19:33.061 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T12:19:33.061 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T12:19:33.061 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T12:19:33.061 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x562949735d00) 2024-09-21T12:19:33.061 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: cache_index_and_filter_blocks: 1 2024-09-21T12:19:33.061 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T12:19:33.061 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T12:19:33.061 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: pin_top_level_index_and_filter: 1 2024-09-21T12:19:33.062 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: index_type: 0 2024-09-21T12:19:33.062 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: data_block_index_type: 0 2024-09-21T12:19:33.062 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: index_shortening: 1 2024-09-21T12:19:33.062 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T12:19:33.062 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: hash_index_allow_collision: 1 2024-09-21T12:19:33.062 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: checksum: 1 2024-09-21T12:19:33.062 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: no_block_cache: 0 2024-09-21T12:19:33.062 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: block_cache: 0x5629497a0170 2024-09-21T12:19:33.062 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: block_cache_name: BinnedLRUCache 2024-09-21T12:19:33.062 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: block_cache_options: 2024-09-21T12:19:33.062 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: capacity : 536870912 2024-09-21T12:19:33.062 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: num_shard_bits : 4 2024-09-21T12:19:33.062 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: strict_capacity_limit : 0 2024-09-21T12:19:33.063 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: high_pri_pool_ratio: 0.000 2024-09-21T12:19:33.063 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: block_cache_compressed: (nil) 2024-09-21T12:19:33.063 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: persistent_cache: (nil) 2024-09-21T12:19:33.063 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: block_size: 4096 2024-09-21T12:19:33.063 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: block_size_deviation: 10 2024-09-21T12:19:33.063 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: block_restart_interval: 16 2024-09-21T12:19:33.063 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: index_block_restart_interval: 1 2024-09-21T12:19:33.063 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: metadata_block_size: 4096 2024-09-21T12:19:33.063 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: partition_filters: 0 2024-09-21T12:19:33.063 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: use_delta_encoding: 1 2024-09-21T12:19:33.063 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-21T12:19:33.063 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: whole_key_filtering: 1 2024-09-21T12:19:33.063 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: verify_compression: 0 2024-09-21T12:19:33.063 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: read_amp_bytes_per_bit: 0 2024-09-21T12:19:33.064 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: format_version: 4 2024-09-21T12:19:33.064 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: enable_index_compression: 1 2024-09-21T12:19:33.064 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: block_align: 0 2024-09-21T12:19:33.064 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T12:19:33.064 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T12:19:33.064 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.compression: NoCompression 2024-09-21T12:19:33.064 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T12:19:33.064 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T12:19:33.064 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T12:19:33.064 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.num_levels: 7 2024-09-21T12:19:33.064 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T12:19:33.064 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T12:19:33.064 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T12:19:33.064 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T12:19:33.065 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T12:19:33.065 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T12:19:33.065 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T12:19:33.065 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T12:19:33.065 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T12:19:33.065 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T12:19:33.065 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T12:19:33.065 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T12:19:33.065 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T12:19:33.065 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T12:19:33.065 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T12:19:33.065 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T12:19:33.065 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T12:19:33.065 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T12:19:33.066 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T12:19:33.066 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T12:19:33.066 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T12:19:33.066 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T12:19:33.066 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T12:19:33.066 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T12:19:33.066 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T12:19:33.066 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T12:19:33.066 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T12:19:33.066 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T12:19:33.066 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T12:19:33.066 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T12:19:33.066 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T12:19:33.066 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T12:19:33.067 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T12:19:33.067 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T12:19:33.067 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.arena_block_size: 4194304 2024-09-21T12:19:33.067 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T12:19:33.067 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T12:19:33.067 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-21T12:19:33.067 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T12:19:33.067 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T12:19:33.067 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T12:19:33.067 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T12:19:33.067 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T12:19:33.067 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T12:19:33.067 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T12:19:33.067 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T12:19:33.068 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T12:19:33.068 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T12:19:33.068 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T12:19:33.068 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.table_properties_collectors: 2024-09-21T12:19:33.068 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.inplace_update_support: 0 2024-09-21T12:19:33.068 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T12:19:33.068 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T12:19:33.068 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T12:19:33.068 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T12:19:33.068 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.bloom_locality: 0 2024-09-21T12:19:33.068 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.max_successive_merges: 0 2024-09-21T12:19:33.068 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T12:19:33.068 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T12:19:33.068 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T12:19:33.069 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T12:19:33.069 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.ttl: 2592000 2024-09-21T12:19:33.069 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T12:19:33.069 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.enable_blob_files: false 2024-09-21T12:19:33.069 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.min_blob_size: 0 2024-09-21T12:19:33.069 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T12:19:33.069 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T12:19:33.069 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T12:19:33.069 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T12:19:33.069 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi148/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T12:19:33.069 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-09-21T12:19:33.069 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-09-21T12:19:33.069 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726921172735974, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-21T12:19:33.069 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-09-21T12:19:33.070 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-21T12:19:33.070 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726921172736402, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726921172, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "99a42100-ffb9-4325-b5a5-b454ddf0bb34", "db_session_id": "GNO3S8BDC11NGZCKD8TU"}} 2024-09-21T12:19:33.070 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-09-21T12:19:33.070 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726921172736824, "job": 1, "event": "recovery_finished"} 2024-09-21T12:19:33.070 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi148/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T12:19:33.070 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x562949786a80 2024-09-21T12:19:33.070 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: DB pointer 0x562949796000 2024-09-21T12:19:33.070 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-21T12:19:33.070 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-21T12:19:33.070 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: ** DB Stats ** 2024-09-21T12:19:33.070 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T12:19:33.070 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T12:19:33.070 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T12:19:33.070 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T12:19:33.071 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T12:19:33.071 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-21T12:19:33.071 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T12:19:33.071 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: 2024-09-21T12:19:33.071 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: ** Compaction Stats [default] ** 2024-09-21T12:19:33.071 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T12:19:33.071 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T12:19:33.071 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-09-21T12:19:33.071 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-09-21T12:19:33.071 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-09-21T12:19:33.071 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: 2024-09-21T12:19:33.071 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: ** Compaction Stats [default] ** 2024-09-21T12:19:33.071 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T12:19:33.071 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T12:19:33.072 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-09-21T12:19:33.072 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T12:19:33.072 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-21T12:19:33.072 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T12:19:33.072 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T12:19:33.072 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T12:19:33.072 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T12:19:33.072 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Cumulative compaction: 0.00 GB write, 0.40 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T12:19:33.072 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Interval compaction: 0.00 GB write, 0.40 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T12:19:33.072 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T12:19:33.072 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: 2024-09-21T12:19:33.072 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T12:19:33.072 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: 2024-09-21T12:19:33.072 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: ** Compaction Stats [default] ** 2024-09-21T12:19:33.073 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T12:19:33.073 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T12:19:33.073 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-09-21T12:19:33.073 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-09-21T12:19:33.073 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-21T12:19:33.073 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: 2024-09-21T12:19:33.073 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: ** Compaction Stats [default] ** 2024-09-21T12:19:33.073 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T12:19:33.073 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T12:19:33.073 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.6 0.00 0.00 1 0.000 0 0 2024-09-21T12:19:33.073 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T12:19:33.073 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-21T12:19:33.073 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T12:19:33.073 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T12:19:33.074 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T12:19:33.074 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T12:19:33.074 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Cumulative compaction: 0.00 GB write, 0.40 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T12:19:33.074 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T12:19:33.074 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T12:19:33.074 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: 2024-09-21T12:19:33.074 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T12:19:33.074 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: mon.smithi148 does not exist in monmap, will attempt to join an existing cluster 2024-09-21T12:19:33.074 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: using public_addr v2:172.21.15.148:0/0 -> [v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0] 2024-09-21T12:19:33.074 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: starting mon.smithi148 rank -1 at public addrs [v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0] at bind addrs [v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi148 fsid 6263deca-7813-11ef-baf6-efdc52797490 2024-09-21T12:19:33.074 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: mon.smithi148@-1(???) e0 preinit fsid 6263deca-7813-11ef-baf6-efdc52797490 2024-09-21T12:19:33.074 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: mon.smithi148@-1(synchronizing).mds e1 new map 2024-09-21T12:19:33.074 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: mon.smithi148@-1(synchronizing).mds e1 print_map 2024-09-21T12:19:33.075 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: e1 2024-09-21T12:19:33.075 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T12:19:33.075 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T12:19:33.075 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: legacy client fscid: -1 2024-09-21T12:19:33.075 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: 2024-09-21T12:19:33.075 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: No filesystems configured 2024-09-21T12:19:33.075 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: mon.smithi148@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-21T12:19:33.075 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: mon.smithi148@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-21T12:19:33.075 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: mon.smithi148@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-21T12:19:33.075 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: mon.smithi148@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-21T12:19:33.075 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: mon.smithi148@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-21T12:19:33.075 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: mon.smithi148@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-21T12:19:33.075 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: mon.smithi148@-1(synchronizing).osd e5 e5: 0 total, 0 up, 0 in 2024-09-21T12:19:33.075 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: mon.smithi148@-1(synchronizing).osd e5 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-21T12:19:33.076 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: mon.smithi148@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T12:19:33.076 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: mon.smithi148@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T12:19:33.076 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: mon.smithi148@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T12:19:33.076 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:33.076 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi148", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:19:33.076 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: Deploying cephadm binary to smithi148 2024-09-21T12:19:33.076 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:33.076 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: Added host smithi148 2024-09-21T12:19:33.076 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:33.076 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:33.076 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T12:19:33.076 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:33.076 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:33.076 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-21T12:19:33.077 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-21T12:19:33.077 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:19:33.077 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: Deploying daemon crash.smithi078 on smithi078 2024-09-21T12:19:33.077 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:33.077 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/3791962968' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-21T12:19:33.077 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:33.077 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:33.077 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:33.077 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T12:19:33.077 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:33.077 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T12:19:33.077 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: Deploying daemon grafana.smithi078 on smithi078 2024-09-21T12:19:33.077 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:33.078 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/3791962968' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-21T12:19:33.078 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T12:19:33.078 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:33.078 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:33.078 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:33.078 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='client.14194 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi078:172.21.15.78=smithi078;smithi148:172.21.15.148=smithi148", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:19:33.078 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: Saving service mon spec with placement smithi078:172.21.15.78=smithi078;smithi148:172.21.15.148=smithi148;count:2 2024-09-21T12:19:33.078 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:33.078 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='client.? 172.21.15.148:0/1747267838' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T12:19:33.078 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:33.078 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:33.078 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='client.? 172.21.15.148:0/3816282307' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T12:19:33.078 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:33.079 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:33.079 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: Deploying daemon node-exporter.smithi078 on smithi078 2024-09-21T12:19:33.079 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='client.? 172.21.15.148:0/434495684' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T12:19:33.079 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:33.079 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:33.079 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-21T12:19:33.079 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:33.079 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14162 172.21.15.78:0/2028727462' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-21T12:19:33.079 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: mgrmap e14: smithi078.ntwyan(active, since 45s) 2024-09-21T12:19:33.079 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='client.? 172.21.15.148:0/379503574' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T12:19:33.079 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='client.? 172.21.15.148:0/3021242459' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T12:19:33.079 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: Active manager daemon smithi078.ntwyan restarted 2024-09-21T12:19:33.079 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: Activating manager daemon smithi078.ntwyan 2024-09-21T12:19:33.079 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: osdmap e5: 0 total, 0 up, 0 in 2024-09-21T12:19:33.080 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: mgrmap e15: smithi078.ntwyan(active, starting, since 0.052298s) 2024-09-21T12:19:33.080 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-21T12:19:33.080 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mgr metadata", "who": "smithi078.ntwyan", "id": "smithi078.ntwyan"}]: dispatch 2024-09-21T12:19:33.080 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T12:19:33.080 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T12:19:33.080 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T12:19:33.080 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: Manager daemon smithi078.ntwyan is now available 2024-09-21T12:19:33.080 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:19:33.080 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:19:33.080 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:19:33.080 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:19:33.080 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:33.080 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/mirror_snapshot_schedule"}]: dispatch 2024-09-21T12:19:33.080 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/trash_purge_schedule"}]: dispatch 2024-09-21T12:19:33.081 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:33.081 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:33.081 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: [21/Sep/2024:12:19:14] ENGINE Bus STARTING 2024-09-21T12:19:33.081 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: [21/Sep/2024:12:19:14] ENGINE Serving on https://172.21.15.78:7150 2024-09-21T12:19:33.081 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: [21/Sep/2024:12:19:14] ENGINE Bus STARTED 2024-09-21T12:19:33.081 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:33.081 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: mgrmap e16: smithi078.ntwyan(active, since 1.05682s) 2024-09-21T12:19:33.081 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='client.? 172.21.15.148:0/1896679403' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T12:19:33.081 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:33.081 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: mgrmap e17: smithi078.ntwyan(active, since 2s) 2024-09-21T12:19:33.081 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:33.081 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:33.081 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd/host:smithi078", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:19:33.081 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:33.082 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: Updating smithi078:/etc/ceph/ceph.conf 2024-09-21T12:19:33.082 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-09-21T12:19:33.082 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='client.? 172.21.15.148:0/4258519499' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T12:19:33.082 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:33.082 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:33.082 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd/host:smithi148", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:19:33.082 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:33.082 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi148", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-21T12:19:33.082 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi148", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-21T12:19:33.082 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:19:33.082 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: Deploying daemon crash.smithi148 on smithi148 2024-09-21T12:19:33.082 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='client.? 172.21.15.148:0/2572010379' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T12:19:33.082 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:33.082 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi148.oyiqmx", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T12:19:33.083 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi148.oyiqmx", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-21T12:19:33.083 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T12:19:33.083 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:19:33.083 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: Deploying daemon mgr.smithi148.oyiqmx on smithi148 2024-09-21T12:19:33.083 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:33.083 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='client.? 172.21.15.148:0/527304142' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T12:19:33.083 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:33.083 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T12:19:33.083 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:19:33.083 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: Deploying daemon mon.smithi148 on smithi148 2024-09-21T12:19:33.083 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:33.083 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: from='client.? 172.21.15.148:0/3700635901' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T12:19:33.083 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: mon.smithi148@-1(synchronizing).paxosservice(auth 1..6) refresh upgraded, format 0 -> 3 2024-09-21T12:19:33.083 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 ceph-mon[25818]: mon.smithi148@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-09-21T12:19:33.084 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 bash[25776]: fda961db230f48e5e21a74a156bb3cd9e8976d6341334f63421eb9aeb8f698fb 2024-09-21T12:19:33.084 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:32 smithi148 systemd[1]: Started Ceph mon.smithi148 for 6263deca-7813-11ef-baf6-efdc52797490. 2024-09-21T12:19:33.372 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-21T12:19:33.372 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T12:19:38.066 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:37 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-21T12:19:38.066 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:37 smithi148 ceph-mon[25818]: mon.smithi078 calling monitor election 2024-09-21T12:19:38.066 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:37 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi148"}]: dispatch 2024-09-21T12:19:38.066 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:37 smithi148 ceph-mon[25818]: from='mgr.? 172.21.15.148:0/2542821858' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi148.oyiqmx/crt"}]: dispatch 2024-09-21T12:19:38.066 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:37 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi148"}]: dispatch 2024-09-21T12:19:38.066 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:37 smithi148 ceph-mon[25818]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:38.066 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:37 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi148"}]: dispatch 2024-09-21T12:19:38.066 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:37 smithi148 ceph-mon[25818]: mon.smithi148 calling monitor election 2024-09-21T12:19:38.067 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:37 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi148"}]: dispatch 2024-09-21T12:19:38.067 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:37 smithi148 ceph-mon[25818]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:38.067 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:37 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi148"}]: dispatch 2024-09-21T12:19:38.067 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:37 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi148"}]: dispatch 2024-09-21T12:19:38.067 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:37 smithi148 ceph-mon[25818]: mon.smithi078 is new leader, mons smithi078,smithi148 in quorum (ranks 0,1) 2024-09-21T12:19:38.067 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:37 smithi148 ceph-mon[25818]: monmap e2: 2 mons at {smithi078=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0],smithi148=[v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0]} 2024-09-21T12:19:38.067 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:37 smithi148 ceph-mon[25818]: fsmap 2024-09-21T12:19:38.067 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:37 smithi148 ceph-mon[25818]: osdmap e5: 0 total, 0 up, 0 in 2024-09-21T12:19:38.067 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:37 smithi148 ceph-mon[25818]: mgrmap e17: smithi078.ntwyan(active, since 23s) 2024-09-21T12:19:38.067 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:37 smithi148 ceph-mon[25818]: Standby manager daemon smithi148.oyiqmx started 2024-09-21T12:19:38.067 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:37 smithi148 ceph-mon[25818]: from='mgr.? 172.21.15.148:0/2542821858' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T12:19:38.067 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:37 smithi148 ceph-mon[25818]: overall HEALTH_OK 2024-09-21T12:19:38.067 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:37 smithi148 ceph-mon[25818]: from='mgr.? 172.21.15.148:0/2542821858' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi148.oyiqmx/key"}]: dispatch 2024-09-21T12:19:38.067 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:37 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:38.068 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:37 smithi148 ceph-mon[25818]: from='mgr.? 172.21.15.148:0/2542821858' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T12:19:38.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:37 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-21T12:19:38.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:37 smithi078 ceph-mon[23681]: mon.smithi078 calling monitor election 2024-09-21T12:19:38.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:37 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi148"}]: dispatch 2024-09-21T12:19:38.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:37 smithi078 ceph-mon[23681]: from='mgr.? 172.21.15.148:0/2542821858' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi148.oyiqmx/crt"}]: dispatch 2024-09-21T12:19:38.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:37 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi148"}]: dispatch 2024-09-21T12:19:38.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:37 smithi078 ceph-mon[23681]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:38.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:37 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi148"}]: dispatch 2024-09-21T12:19:38.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:37 smithi078 ceph-mon[23681]: mon.smithi148 calling monitor election 2024-09-21T12:19:38.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:37 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi148"}]: dispatch 2024-09-21T12:19:38.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:37 smithi078 ceph-mon[23681]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:38.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:37 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi148"}]: dispatch 2024-09-21T12:19:38.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:37 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi148"}]: dispatch 2024-09-21T12:19:38.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:37 smithi078 ceph-mon[23681]: mon.smithi078 is new leader, mons smithi078,smithi148 in quorum (ranks 0,1) 2024-09-21T12:19:38.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:37 smithi078 ceph-mon[23681]: monmap e2: 2 mons at {smithi078=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0],smithi148=[v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0]} 2024-09-21T12:19:38.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:37 smithi078 ceph-mon[23681]: fsmap 2024-09-21T12:19:38.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:37 smithi078 ceph-mon[23681]: osdmap e5: 0 total, 0 up, 0 in 2024-09-21T12:19:38.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:37 smithi078 ceph-mon[23681]: mgrmap e17: smithi078.ntwyan(active, since 23s) 2024-09-21T12:19:38.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:37 smithi078 ceph-mon[23681]: Standby manager daemon smithi148.oyiqmx started 2024-09-21T12:19:38.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:37 smithi078 ceph-mon[23681]: from='mgr.? 172.21.15.148:0/2542821858' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T12:19:38.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:37 smithi078 ceph-mon[23681]: overall HEALTH_OK 2024-09-21T12:19:38.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:37 smithi078 ceph-mon[23681]: from='mgr.? 172.21.15.148:0/2542821858' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi148.oyiqmx/key"}]: dispatch 2024-09-21T12:19:38.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:37 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:38.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:37 smithi078 ceph-mon[23681]: from='mgr.? 172.21.15.148:0/2542821858' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T12:19:38.440 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-09-21T12:19:38.441 INFO:teuthology.orchestra.run.smithi148.stdout:{"epoch":2,"fsid":"6263deca-7813-11ef-baf6-efdc52797490","modified":"2024-09-21T12:19:32.763829Z","created":"2024-09-21T12:17:23.120662Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi148","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:3300","nonce":0},{"type":"v1","addr":"172.21.15.148:6789","nonce":0}]},"addr":"172.21.15.148:6789/0","public_addr":"172.21.15.148:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-21T12:19:38.442 INFO:teuthology.orchestra.run.smithi148.stderr:dumped monmap epoch 2 2024-09-21T12:19:39.074 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:38 smithi148 ceph-mon[25818]: Deploying daemon node-exporter.smithi148 on smithi148 2024-09-21T12:19:39.074 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:38 smithi148 ceph-mon[25818]: mgrmap e18: smithi078.ntwyan(active, since 23s), standbys: smithi148.oyiqmx 2024-09-21T12:19:39.074 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:38 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mgr metadata", "who": "smithi148.oyiqmx", "id": "smithi148.oyiqmx"}]: dispatch 2024-09-21T12:19:39.074 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:38 smithi148 ceph-mon[25818]: from='client.? 172.21.15.148:0/3512736104' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T12:19:39.074 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:38 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi148"}]: dispatch 2024-09-21T12:19:39.076 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-21T12:19:39.076 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph config generate-minimal-conf 2024-09-21T12:19:39.102 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:38 smithi078 ceph-mon[23681]: Deploying daemon node-exporter.smithi148 on smithi148 2024-09-21T12:19:39.102 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:38 smithi078 ceph-mon[23681]: mgrmap e18: smithi078.ntwyan(active, since 23s), standbys: smithi148.oyiqmx 2024-09-21T12:19:39.102 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:38 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mgr metadata", "who": "smithi148.oyiqmx", "id": "smithi148.oyiqmx"}]: dispatch 2024-09-21T12:19:39.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:38 smithi078 ceph-mon[23681]: from='client.? 172.21.15.148:0/3512736104' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T12:19:39.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:38 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi148"}]: dispatch 2024-09-21T12:19:40.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:39 smithi078 ceph-mon[23681]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:40.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:39 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:40.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:39 smithi148 ceph-mon[25818]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:40.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:39 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:41.069 INFO:teuthology.orchestra.run.smithi078.stdout:# minimal ceph.conf for 6263deca-7813-11ef-baf6-efdc52797490 2024-09-21T12:19:41.069 INFO:teuthology.orchestra.run.smithi078.stdout:[global] 2024-09-21T12:19:41.069 INFO:teuthology.orchestra.run.smithi078.stdout: fsid = 6263deca-7813-11ef-baf6-efdc52797490 2024-09-21T12:19:41.069 INFO:teuthology.orchestra.run.smithi078.stdout: mon_host = [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] [v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0] 2024-09-21T12:19:41.667 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-21T12:19:41.667 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T12:19:41.667 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/ceph/ceph.conf 2024-09-21T12:19:41.735 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T12:19:41.735 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-21T12:19:41.799 DEBUG:teuthology.orchestra.run.smithi148:> set -ex 2024-09-21T12:19:41.799 DEBUG:teuthology.orchestra.run.smithi148:> sudo dd of=/etc/ceph/ceph.conf 2024-09-21T12:19:41.835 DEBUG:teuthology.orchestra.run.smithi148:> set -ex 2024-09-21T12:19:41.836 DEBUG:teuthology.orchestra.run.smithi148:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-21T12:19:41.902 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:41 smithi078 ceph-mon[23681]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:41.902 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:41 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/1303762083' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:19:41.903 DEBUG:tasks.cephadm:set 0 configs 2024-09-21T12:19:41.903 INFO:tasks.cephadm:Deploying OSDs... 2024-09-21T12:19:41.903 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T12:19:41.903 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/scratch_devs of=/dev/stdout 2024-09-21T12:19:41.919 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-21T12:19:41.919 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_1 2024-09-21T12:19:41.975 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-21T12:19:41.975 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T12:19:41.975 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-09-21T12:19:41.975 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T12:19:41.975 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T12:19:41.976 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-09-21 12:19:04.551585965 +0000 2024-09-21T12:19:41.976 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-09-21 12:16:31.981903435 +0000 2024-09-21T12:19:41.976 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-09-21 12:16:31.981903435 +0000 2024-09-21T12:19:41.976 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-09-21 12:16:31.981903435 +0000 2024-09-21T12:19:41.976 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-21T12:19:42.039 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-09-21T12:19:42.039 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-09-21T12:19:42.039 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000256553 s, 2.0 MB/s 2024-09-21T12:19:42.040 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-21T12:19:42.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:41 smithi148 ceph-mon[25818]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:42.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:41 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/1303762083' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:19:42.097 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_2 2024-09-21T12:19:42.154 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-21T12:19:42.155 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T12:19:42.155 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-09-21T12:19:42.155 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T12:19:42.155 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T12:19:42.155 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-09-21 12:19:04.551585965 +0000 2024-09-21T12:19:42.155 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-09-21 12:16:32.313898118 +0000 2024-09-21T12:19:42.155 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-09-21 12:16:32.313898118 +0000 2024-09-21T12:19:42.155 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-09-21 12:16:32.313898118 +0000 2024-09-21T12:19:42.155 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-21T12:19:42.218 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-09-21T12:19:42.218 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-09-21T12:19:42.218 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000168134 s, 3.0 MB/s 2024-09-21T12:19:42.220 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-21T12:19:42.276 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_3 2024-09-21T12:19:42.331 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-21T12:19:42.331 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T12:19:42.331 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-09-21T12:19:42.331 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T12:19:42.331 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T12:19:42.332 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-09-21 12:19:04.551585965 +0000 2024-09-21T12:19:42.332 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-09-21 12:16:32.638892914 +0000 2024-09-21T12:19:42.332 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-09-21 12:16:32.638892914 +0000 2024-09-21T12:19:42.332 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-09-21 12:16:32.638892914 +0000 2024-09-21T12:19:42.332 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-21T12:19:42.395 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-09-21T12:19:42.396 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-09-21T12:19:42.396 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000175922 s, 2.9 MB/s 2024-09-21T12:19:42.397 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-21T12:19:42.453 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_4 2024-09-21T12:19:42.509 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-21T12:19:42.509 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T12:19:42.509 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-09-21T12:19:42.509 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T12:19:42.509 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T12:19:42.509 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-09-21 12:19:04.552585949 +0000 2024-09-21T12:19:42.509 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-09-21 12:16:32.959887773 +0000 2024-09-21T12:19:42.509 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-09-21 12:16:32.959887773 +0000 2024-09-21T12:19:42.509 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-09-21 12:16:32.959887773 +0000 2024-09-21T12:19:42.510 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-21T12:19:42.572 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-09-21T12:19:42.572 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-09-21T12:19:42.572 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000206061 s, 2.5 MB/s 2024-09-21T12:19:42.573 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-21T12:19:42.630 DEBUG:teuthology.orchestra.run.smithi148:> set -ex 2024-09-21T12:19:42.630 DEBUG:teuthology.orchestra.run.smithi148:> dd if=/scratch_devs of=/dev/stdout 2024-09-21T12:19:42.649 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-21T12:19:42.650 DEBUG:teuthology.orchestra.run.smithi148:> stat /dev/vg_nvme/lv_1 2024-09-21T12:19:42.709 INFO:teuthology.orchestra.run.smithi148.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-21T12:19:42.709 INFO:teuthology.orchestra.run.smithi148.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T12:19:42.709 INFO:teuthology.orchestra.run.smithi148.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-09-21T12:19:42.709 INFO:teuthology.orchestra.run.smithi148.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T12:19:42.709 INFO:teuthology.orchestra.run.smithi148.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T12:19:42.710 INFO:teuthology.orchestra.run.smithi148.stdout:Access: 2024-09-21 12:19:38.433132273 +0000 2024-09-21T12:19:42.710 INFO:teuthology.orchestra.run.smithi148.stdout:Modify: 2024-09-21 12:16:31.725556838 +0000 2024-09-21T12:19:42.710 INFO:teuthology.orchestra.run.smithi148.stdout:Change: 2024-09-21 12:16:31.725556838 +0000 2024-09-21T12:19:42.710 INFO:teuthology.orchestra.run.smithi148.stdout: Birth: 2024-09-21 12:16:31.725556838 +0000 2024-09-21T12:19:42.710 DEBUG:teuthology.orchestra.run.smithi148:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-21T12:19:42.778 INFO:teuthology.orchestra.run.smithi148.stderr:1+0 records in 2024-09-21T12:19:42.778 INFO:teuthology.orchestra.run.smithi148.stderr:1+0 records out 2024-09-21T12:19:42.778 INFO:teuthology.orchestra.run.smithi148.stderr:512 bytes copied, 0.000235536 s, 2.2 MB/s 2024-09-21T12:19:42.779 DEBUG:teuthology.orchestra.run.smithi148:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-21T12:19:42.838 DEBUG:teuthology.orchestra.run.smithi148:> stat /dev/vg_nvme/lv_2 2024-09-21T12:19:42.895 INFO:teuthology.orchestra.run.smithi148.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-21T12:19:42.895 INFO:teuthology.orchestra.run.smithi148.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T12:19:42.895 INFO:teuthology.orchestra.run.smithi148.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-09-21T12:19:42.895 INFO:teuthology.orchestra.run.smithi148.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T12:19:42.895 INFO:teuthology.orchestra.run.smithi148.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T12:19:42.895 INFO:teuthology.orchestra.run.smithi148.stdout:Access: 2024-09-21 12:19:38.434132249 +0000 2024-09-21T12:19:42.896 INFO:teuthology.orchestra.run.smithi148.stdout:Modify: 2024-09-21 12:16:32.057548720 +0000 2024-09-21T12:19:42.896 INFO:teuthology.orchestra.run.smithi148.stdout:Change: 2024-09-21 12:16:32.057548720 +0000 2024-09-21T12:19:42.896 INFO:teuthology.orchestra.run.smithi148.stdout: Birth: 2024-09-21 12:16:32.057548720 +0000 2024-09-21T12:19:42.896 DEBUG:teuthology.orchestra.run.smithi148:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-21T12:19:42.960 INFO:teuthology.orchestra.run.smithi148.stderr:1+0 records in 2024-09-21T12:19:42.960 INFO:teuthology.orchestra.run.smithi148.stderr:1+0 records out 2024-09-21T12:19:42.960 INFO:teuthology.orchestra.run.smithi148.stderr:512 bytes copied, 0.000205834 s, 2.5 MB/s 2024-09-21T12:19:42.962 DEBUG:teuthology.orchestra.run.smithi148:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-21T12:19:43.020 DEBUG:teuthology.orchestra.run.smithi148:> stat /dev/vg_nvme/lv_3 2024-09-21T12:19:43.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:42 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:43.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:42 smithi148 ceph-mon[25818]: Deploying daemon prometheus.smithi078 on smithi078 2024-09-21T12:19:43.066 INFO:teuthology.orchestra.run.smithi148.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-21T12:19:43.066 INFO:teuthology.orchestra.run.smithi148.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T12:19:43.066 INFO:teuthology.orchestra.run.smithi148.stdout:Device: 5h/5d Inode: 704 Links: 1 2024-09-21T12:19:43.066 INFO:teuthology.orchestra.run.smithi148.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T12:19:43.066 INFO:teuthology.orchestra.run.smithi148.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T12:19:43.066 INFO:teuthology.orchestra.run.smithi148.stdout:Access: 2024-09-21 12:19:38.434132249 +0000 2024-09-21T12:19:43.067 INFO:teuthology.orchestra.run.smithi148.stdout:Modify: 2024-09-21 12:16:32.387540652 +0000 2024-09-21T12:19:43.067 INFO:teuthology.orchestra.run.smithi148.stdout:Change: 2024-09-21 12:16:32.387540652 +0000 2024-09-21T12:19:43.067 INFO:teuthology.orchestra.run.smithi148.stdout: Birth: 2024-09-21 12:16:32.387540652 +0000 2024-09-21T12:19:43.067 DEBUG:teuthology.orchestra.run.smithi148:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-21T12:19:43.130 INFO:teuthology.orchestra.run.smithi148.stderr:1+0 records in 2024-09-21T12:19:43.130 INFO:teuthology.orchestra.run.smithi148.stderr:1+0 records out 2024-09-21T12:19:43.131 INFO:teuthology.orchestra.run.smithi148.stderr:512 bytes copied, 0.000224059 s, 2.3 MB/s 2024-09-21T12:19:43.132 DEBUG:teuthology.orchestra.run.smithi148:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-21T12:19:43.189 DEBUG:teuthology.orchestra.run.smithi148:> stat /dev/vg_nvme/lv_4 2024-09-21T12:19:43.245 INFO:teuthology.orchestra.run.smithi148.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-21T12:19:43.245 INFO:teuthology.orchestra.run.smithi148.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T12:19:43.245 INFO:teuthology.orchestra.run.smithi148.stdout:Device: 5h/5d Inode: 725 Links: 1 2024-09-21T12:19:43.245 INFO:teuthology.orchestra.run.smithi148.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T12:19:43.245 INFO:teuthology.orchestra.run.smithi148.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T12:19:43.246 INFO:teuthology.orchestra.run.smithi148.stdout:Access: 2024-09-21 12:19:38.435132225 +0000 2024-09-21T12:19:43.246 INFO:teuthology.orchestra.run.smithi148.stdout:Modify: 2024-09-21 12:16:32.714532656 +0000 2024-09-21T12:19:43.246 INFO:teuthology.orchestra.run.smithi148.stdout:Change: 2024-09-21 12:16:32.714532656 +0000 2024-09-21T12:19:43.246 INFO:teuthology.orchestra.run.smithi148.stdout: Birth: 2024-09-21 12:16:32.714532656 +0000 2024-09-21T12:19:43.246 DEBUG:teuthology.orchestra.run.smithi148:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-21T12:19:43.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:42 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:43.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:42 smithi078 ceph-mon[23681]: Deploying daemon prometheus.smithi078 on smithi078 2024-09-21T12:19:43.309 INFO:teuthology.orchestra.run.smithi148.stderr:1+0 records in 2024-09-21T12:19:43.309 INFO:teuthology.orchestra.run.smithi148.stderr:1+0 records out 2024-09-21T12:19:43.309 INFO:teuthology.orchestra.run.smithi148.stderr:512 bytes copied, 0.000189413 s, 2.7 MB/s 2024-09-21T12:19:43.310 DEBUG:teuthology.orchestra.run.smithi148:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-21T12:19:43.366 INFO:tasks.cephadm:Deploying osd.0 on smithi078 with /dev/vg_nvme/lv_4... 2024-09-21T12:19:43.367 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_4 2024-09-21T12:19:44.166 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:43 smithi078 ceph-mon[23681]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:44.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:43 smithi148 ceph-mon[25818]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:45.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:45 smithi148 ceph-mon[25818]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:45.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:45 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:45.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:45 smithi078 ceph-mon[23681]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:45.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:45 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:47.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:47 smithi148 ceph-mon[25818]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:47.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:47 smithi078 ceph-mon[23681]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:49.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:49 smithi148 ceph-mon[25818]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:49.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:49 smithi078 ceph-mon[23681]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:51.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:51 smithi078 ceph-mon[23681]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:51.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:51 smithi148 ceph-mon[25818]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:53.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:53 smithi078 ceph-mon[23681]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:53.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:53 smithi148 ceph-mon[25818]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:55.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:55 smithi078 ceph-mon[23681]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:55.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:55 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:55.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:55 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:19:55.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:55 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:19:55.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:55 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:19:55.392 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:55 smithi148 ceph-mon[25818]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:55.392 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:55 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:55.392 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:55 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:19:55.393 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:55 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:19:55.393 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:55 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:19:55.608 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T12:19:55.627 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi078:vg_nvme/lv_4 2024-09-21T12:19:57.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:56 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:57.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:56 smithi148 ceph-mon[25818]: Updating smithi148:/etc/ceph/ceph.conf 2024-09-21T12:19:57.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:56 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:57.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:56 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:57.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:56 smithi078 ceph-mon[23681]: Updating smithi148:/etc/ceph/ceph.conf 2024-09-21T12:19:57.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:56 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:19:58.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:57 smithi148 ceph-mon[25818]: Updating smithi148:/etc/ceph/ceph.client.admin.keyring 2024-09-21T12:19:58.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:57 smithi148 ceph-mon[25818]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:58.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:57 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T12:19:58.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:57 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T12:19:58.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:57 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:19:58.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:57 smithi078 ceph-mon[23681]: Updating smithi148:/etc/ceph/ceph.client.admin.keyring 2024-09-21T12:19:58.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:57 smithi078 ceph-mon[23681]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:19:58.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:57 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T12:19:58.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:57 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T12:19:58.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:57 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:19:59.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:58 smithi148 ceph-mon[25818]: from='client.14236 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:19:59.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:58 smithi078 ceph-mon[23681]: from='client.14236 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:20:00.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:59 smithi148 ceph-mon[25818]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:20:00.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:19:59 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:00.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:59 smithi078 ceph-mon[23681]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:20:00.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:19:59 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:01.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:00 smithi078 ceph-mon[23681]: overall HEALTH_OK 2024-09-21T12:20:01.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:00 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/4089086857' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "908d7633-5010-4ace-8275-e8e2c504b0f7"}]: dispatch 2024-09-21T12:20:01.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:00 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/4089086857' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "908d7633-5010-4ace-8275-e8e2c504b0f7"}]': finished 2024-09-21T12:20:01.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:00 smithi078 ceph-mon[23681]: osdmap e6: 1 total, 0 up, 1 in 2024-09-21T12:20:01.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:00 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T12:20:01.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:00 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/3411402108' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T12:20:01.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:00 smithi148 ceph-mon[25818]: overall HEALTH_OK 2024-09-21T12:20:01.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:00 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/4089086857' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "908d7633-5010-4ace-8275-e8e2c504b0f7"}]: dispatch 2024-09-21T12:20:01.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:00 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/4089086857' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "908d7633-5010-4ace-8275-e8e2c504b0f7"}]': finished 2024-09-21T12:20:01.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:00 smithi148 ceph-mon[25818]: osdmap e6: 1 total, 0 up, 1 in 2024-09-21T12:20:01.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:00 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T12:20:01.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:00 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/3411402108' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T12:20:02.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:01 smithi078 ceph-mon[23681]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:20:02.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:01 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:02.201 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:01 smithi148 ceph-mon[25818]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:20:02.202 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:01 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:03.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:02 smithi148 ceph-mon[25818]: Updating smithi078:/etc/ceph/ceph.conf 2024-09-21T12:20:03.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:02 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:03.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:02 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:03.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:02 smithi148 ceph-mon[25818]: Reconfiguring mon.smithi078 (unknown last config time)... 2024-09-21T12:20:03.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:02 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T12:20:03.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:02 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T12:20:03.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:02 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:20:03.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:02 smithi148 ceph-mon[25818]: Reconfiguring daemon mon.smithi078 on smithi078 2024-09-21T12:20:03.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:02 smithi078 ceph-mon[23681]: Updating smithi078:/etc/ceph/ceph.conf 2024-09-21T12:20:03.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:02 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:03.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:02 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:03.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:02 smithi078 ceph-mon[23681]: Reconfiguring mon.smithi078 (unknown last config time)... 2024-09-21T12:20:03.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:02 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T12:20:03.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:02 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T12:20:03.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:02 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:20:03.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:02 smithi078 ceph-mon[23681]: Reconfiguring daemon mon.smithi078 on smithi078 2024-09-21T12:20:04.199 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:03 smithi078 ceph-mon[23681]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:20:04.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:03 smithi148 ceph-mon[25818]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:20:06.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:05 smithi078 ceph-mon[23681]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:20:06.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:05 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:06.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:05 smithi078 ceph-mon[23681]: Reconfiguring mgr.smithi078.ntwyan (unknown last config time)... 2024-09-21T12:20:06.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:05 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.ntwyan", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T12:20:06.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:05 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T12:20:06.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:05 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:20:06.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:05 smithi078 ceph-mon[23681]: Reconfiguring daemon mgr.smithi078.ntwyan on smithi078 2024-09-21T12:20:06.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:05 smithi148 ceph-mon[25818]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:20:06.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:05 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:06.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:05 smithi148 ceph-mon[25818]: Reconfiguring mgr.smithi078.ntwyan (unknown last config time)... 2024-09-21T12:20:06.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:05 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.ntwyan", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T12:20:06.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:05 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T12:20:06.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:05 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:20:06.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:05 smithi148 ceph-mon[25818]: Reconfiguring daemon mgr.smithi078.ntwyan on smithi078 2024-09-21T12:20:08.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:07 smithi148 ceph-mon[25818]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:20:08.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:07 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:08.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:07 smithi078 ceph-mon[23681]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:20:08.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:07 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:09.130 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:08 smithi078 ceph-mon[23681]: Reconfiguring alertmanager.smithi078 (dependencies changed)... 2024-09-21T12:20:09.130 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:08 smithi078 ceph-mon[23681]: Reconfiguring daemon alertmanager.smithi078 on smithi078 2024-09-21T12:20:09.130 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:08 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-21T12:20:09.130 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:08 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:20:09.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:08 smithi148 ceph-mon[25818]: Reconfiguring alertmanager.smithi078 (dependencies changed)... 2024-09-21T12:20:09.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:08 smithi148 ceph-mon[25818]: Reconfiguring daemon alertmanager.smithi078 on smithi078 2024-09-21T12:20:09.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:08 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-21T12:20:09.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:08 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:20:10.210 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:09 smithi078 ceph-mon[23681]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:20:10.210 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:09 smithi078 ceph-mon[23681]: Deploying daemon osd.0 on smithi078 2024-09-21T12:20:10.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:09 smithi148 ceph-mon[25818]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:20:10.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:09 smithi148 ceph-mon[25818]: Deploying daemon osd.0 on smithi078 2024-09-21T12:20:12.239 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:11 smithi078 ceph-mon[23681]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:20:12.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:11 smithi148 ceph-mon[25818]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:20:14.230 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:13 smithi078 ceph-mon[23681]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:20:14.230 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:13 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:14.231 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:13 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-21T12:20:14.231 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:13 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:20:14.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:13 smithi148 ceph-mon[25818]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:20:14.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:13 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:14.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:13 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-21T12:20:14.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:13 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:20:15.283 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:14 smithi078 ceph-mon[23681]: Reconfiguring crash.smithi078 (monmap changed)... 2024-09-21T12:20:15.283 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:14 smithi078 ceph-mon[23681]: Reconfiguring daemon crash.smithi078 on smithi078 2024-09-21T12:20:15.283 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/mirror_snapshot_schedule"}]: dispatch 2024-09-21T12:20:15.283 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/trash_purge_schedule"}]: dispatch 2024-09-21T12:20:15.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:14 smithi148 ceph-mon[25818]: Reconfiguring crash.smithi078 (monmap changed)... 2024-09-21T12:20:15.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:14 smithi148 ceph-mon[25818]: Reconfiguring daemon crash.smithi078 on smithi078 2024-09-21T12:20:15.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:14 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/mirror_snapshot_schedule"}]: dispatch 2024-09-21T12:20:15.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:14 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/trash_purge_schedule"}]: dispatch 2024-09-21T12:20:16.211 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:15 smithi078 ceph-mon[23681]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:20:16.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:15 smithi148 ceph-mon[25818]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:20:18.200 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:17 smithi078 ceph-mon[23681]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:20:18.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:17 smithi148 ceph-mon[25818]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:20:19.554 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:19 smithi078 ceph-mon[23681]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:20:19.554 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:19 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:19.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:19 smithi148 ceph-mon[25818]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:20:19.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:19 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:20.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:20 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:20.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:20 smithi078 ceph-mon[23681]: Reconfiguring grafana.smithi078 (dependencies changed)... 2024-09-21T12:20:20.494 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:20 smithi078 ceph-mon[23681]: Reconfiguring daemon grafana.smithi078 on smithi078 2024-09-21T12:20:20.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:20 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:20.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:20 smithi148 ceph-mon[25818]: Reconfiguring grafana.smithi078 (dependencies changed)... 2024-09-21T12:20:20.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:20 smithi148 ceph-mon[25818]: Reconfiguring daemon grafana.smithi078 on smithi078 2024-09-21T12:20:21.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:21 smithi148 ceph-mon[25818]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:20:21.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:21 smithi078 ceph-mon[23681]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:20:22.460 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:22 smithi078 ceph-mon[23681]: from='osd.0 [v2:172.21.15.78:6802/2381128776,v1:172.21.15.78:6803/2381128776]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T12:20:22.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:22 smithi148 ceph-mon[25818]: from='osd.0 [v2:172.21.15.78:6802/2381128776,v1:172.21.15.78:6803/2381128776]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T12:20:23.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:23 smithi078 ceph-mon[23681]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:20:23.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:23 smithi078 ceph-mon[23681]: from='osd.0 [v2:172.21.15.78:6802/2381128776,v1:172.21.15.78:6803/2381128776]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T12:20:23.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:23 smithi078 ceph-mon[23681]: osdmap e7: 1 total, 0 up, 1 in 2024-09-21T12:20:23.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:23 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T12:20:23.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:23 smithi078 ceph-mon[23681]: from='osd.0 [v2:172.21.15.78:6802/2381128776,v1:172.21.15.78:6803/2381128776]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-09-21T12:20:23.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:23 smithi148 ceph-mon[25818]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:20:23.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:23 smithi148 ceph-mon[25818]: from='osd.0 [v2:172.21.15.78:6802/2381128776,v1:172.21.15.78:6803/2381128776]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T12:20:23.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:23 smithi148 ceph-mon[25818]: osdmap e7: 1 total, 0 up, 1 in 2024-09-21T12:20:23.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:23 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T12:20:23.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:23 smithi148 ceph-mon[25818]: from='osd.0 [v2:172.21.15.78:6802/2381128776,v1:172.21.15.78:6803/2381128776]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-09-21T12:20:24.517 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:24 smithi078 ceph-mon[23681]: from='osd.0 [v2:172.21.15.78:6802/2381128776,v1:172.21.15.78:6803/2381128776]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-09-21T12:20:24.517 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:24 smithi078 ceph-mon[23681]: osdmap e8: 1 total, 0 up, 1 in 2024-09-21T12:20:24.517 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:24 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T12:20:24.517 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:24 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T12:20:24.703 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:24 smithi148 ceph-mon[25818]: from='osd.0 [v2:172.21.15.78:6802/2381128776,v1:172.21.15.78:6803/2381128776]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-09-21T12:20:24.703 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:24 smithi148 ceph-mon[25818]: osdmap e8: 1 total, 0 up, 1 in 2024-09-21T12:20:24.703 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:24 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T12:20:24.703 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:24 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T12:20:25.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:25 smithi148 ceph-mon[25818]: purged_snaps scrub starts 2024-09-21T12:20:25.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:25 smithi148 ceph-mon[25818]: purged_snaps scrub ok 2024-09-21T12:20:25.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:25 smithi148 ceph-mon[25818]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:20:25.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:25 smithi148 ceph-mon[25818]: osd.0 [v2:172.21.15.78:6802/2381128776,v1:172.21.15.78:6803/2381128776] boot 2024-09-21T12:20:25.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:25 smithi148 ceph-mon[25818]: osdmap e9: 1 total, 1 up, 1 in 2024-09-21T12:20:25.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:25 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T12:20:25.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:25 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:25.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:25 smithi148 ceph-mon[25818]: Reconfiguring crash.smithi148 (monmap changed)... 2024-09-21T12:20:25.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:25 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi148", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-21T12:20:25.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:25 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:20:25.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:25 smithi148 ceph-mon[25818]: Reconfiguring daemon crash.smithi148 on smithi148 2024-09-21T12:20:25.798 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:25 smithi078 ceph-mon[23681]: purged_snaps scrub starts 2024-09-21T12:20:25.798 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:25 smithi078 ceph-mon[23681]: purged_snaps scrub ok 2024-09-21T12:20:25.798 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:25 smithi078 ceph-mon[23681]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T12:20:25.798 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:25 smithi078 ceph-mon[23681]: osd.0 [v2:172.21.15.78:6802/2381128776,v1:172.21.15.78:6803/2381128776] boot 2024-09-21T12:20:25.798 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:25 smithi078 ceph-mon[23681]: osdmap e9: 1 total, 1 up, 1 in 2024-09-21T12:20:25.798 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:25 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T12:20:25.798 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:25 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:25.798 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:25 smithi078 ceph-mon[23681]: Reconfiguring crash.smithi148 (monmap changed)... 2024-09-21T12:20:25.799 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:25 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi148", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-21T12:20:25.799 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:25 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:20:25.799 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:25 smithi078 ceph-mon[23681]: Reconfiguring daemon crash.smithi148 on smithi148 2024-09-21T12:20:26.119 INFO:teuthology.orchestra.run.smithi078.stdout:Created osd(s) 0 on host 'smithi078' 2024-09-21T12:20:26.702 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:26 smithi078 ceph-mon[23681]: osdmap e10: 1 total, 1 up, 1 in 2024-09-21T12:20:26.702 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:26 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:26.702 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:26 smithi078 ceph-mon[23681]: Reconfiguring mgr.smithi148.oyiqmx (monmap changed)... 2024-09-21T12:20:26.702 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:26 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi148.oyiqmx", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T12:20:26.702 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:26 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T12:20:26.702 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:26 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:20:26.702 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:26 smithi078 ceph-mon[23681]: Reconfiguring daemon mgr.smithi148.oyiqmx on smithi148 2024-09-21T12:20:26.703 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:26 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:26.726 DEBUG:teuthology.orchestra.run.smithi078:osd.0> sudo journalctl -f -n 0 -u ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.0.service 2024-09-21T12:20:26.728 INFO:tasks.cephadm:Deploying osd.1 on smithi078 with /dev/vg_nvme/lv_3... 2024-09-21T12:20:26.728 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_3 2024-09-21T12:20:26.802 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:26 smithi148 ceph-mon[25818]: osdmap e10: 1 total, 1 up, 1 in 2024-09-21T12:20:26.802 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:26 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:26.802 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:26 smithi148 ceph-mon[25818]: Reconfiguring mgr.smithi148.oyiqmx (monmap changed)... 2024-09-21T12:20:26.802 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:26 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi148.oyiqmx", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T12:20:26.802 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:26 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T12:20:26.802 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:26 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:20:26.802 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:26 smithi148 ceph-mon[25818]: Reconfiguring daemon mgr.smithi148.oyiqmx on smithi148 2024-09-21T12:20:26.802 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:26 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:28.038 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:27 smithi148 ceph-mon[25818]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:28.038 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:27 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:28.038 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:27 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T12:20:28.038 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:27 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T12:20:28.038 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:27 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:20:28.190 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:27 smithi078 ceph-mon[23681]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:28.190 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:27 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:28.190 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:27 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T12:20:28.190 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:27 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T12:20:28.190 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:27 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:20:29.123 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:28 smithi148 ceph-mon[25818]: Reconfiguring mon.smithi148 (monmap changed)... 2024-09-21T12:20:29.123 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:28 smithi148 ceph-mon[25818]: Reconfiguring daemon mon.smithi148 on smithi148 2024-09-21T12:20:29.123 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:28 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:29.123 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:28 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T12:20:29.123 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:28 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.78:9093"}]: dispatch 2024-09-21T12:20:29.123 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:28 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:29.124 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:28 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T12:20:29.124 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:28 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.78:3000"}]: dispatch 2024-09-21T12:20:29.124 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:28 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:29.124 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:28 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T12:20:29.124 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:28 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.78:9095"}]: dispatch 2024-09-21T12:20:29.124 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:28 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:29.124 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:28 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:20:29.124 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:28 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:20:29.124 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:28 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:20:29.244 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:28 smithi078 ceph-mon[23681]: Reconfiguring mon.smithi148 (monmap changed)... 2024-09-21T12:20:29.244 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:28 smithi078 ceph-mon[23681]: Reconfiguring daemon mon.smithi148 on smithi148 2024-09-21T12:20:29.244 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:28 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:29.244 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:28 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T12:20:29.244 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:28 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.78:9093"}]: dispatch 2024-09-21T12:20:29.244 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:28 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:29.244 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:28 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T12:20:29.244 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:28 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.78:3000"}]: dispatch 2024-09-21T12:20:29.244 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:28 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:29.244 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:28 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T12:20:29.245 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:28 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.78:9095"}]: dispatch 2024-09-21T12:20:29.245 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:28 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:29.245 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:28 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:20:29.245 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:28 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:20:29.245 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:28 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:20:29.532 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T12:20:29.552 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi078:vg_nvme/lv_3 2024-09-21T12:20:30.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:29 smithi148 ceph-mon[25818]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:30.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:29 smithi148 ceph-mon[25818]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T12:20:30.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:29 smithi148 ceph-mon[25818]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.78:9093"}]: dispatch 2024-09-21T12:20:30.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:29 smithi148 ceph-mon[25818]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T12:20:30.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:29 smithi148 ceph-mon[25818]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.78:3000"}]: dispatch 2024-09-21T12:20:30.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:29 smithi148 ceph-mon[25818]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T12:20:30.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:29 smithi148 ceph-mon[25818]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.78:9095"}]: dispatch 2024-09-21T12:20:30.184 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:29 smithi078 ceph-mon[23681]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:30.184 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:29 smithi078 ceph-mon[23681]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T12:20:30.184 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:29 smithi078 ceph-mon[23681]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.78:9093"}]: dispatch 2024-09-21T12:20:30.184 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:29 smithi078 ceph-mon[23681]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T12:20:30.184 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:29 smithi078 ceph-mon[23681]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.78:3000"}]: dispatch 2024-09-21T12:20:30.184 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:29 smithi078 ceph-mon[23681]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T12:20:30.184 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:29 smithi078 ceph-mon[23681]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.78:9095"}]: dispatch 2024-09-21T12:20:31.216 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:31 smithi078 ceph-mon[23681]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:31.216 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:31 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:31.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:31 smithi148 ceph-mon[25818]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:31.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:31 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:33.407 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:33 smithi078 ceph-mon[23681]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:33.408 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:33 smithi078 ceph-mon[23681]: from='client.14254 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:20:33.408 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:33 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T12:20:33.408 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:33 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T12:20:33.408 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:33 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:20:33.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:33 smithi148 ceph-mon[25818]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:33.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:33 smithi148 ceph-mon[25818]: from='client.14254 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:20:33.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:33 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T12:20:33.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:33 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T12:20:33.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:33 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:20:35.504 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:35 smithi078 ceph-mon[23681]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:35.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:35 smithi148 ceph-mon[25818]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:36.538 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:36 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/484522784' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "71d611f1-03c6-471c-baa2-602d6f2bc697"}]: dispatch 2024-09-21T12:20:36.538 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:36 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/484522784' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "71d611f1-03c6-471c-baa2-602d6f2bc697"}]': finished 2024-09-21T12:20:36.538 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:36 smithi078 ceph-mon[23681]: osdmap e11: 2 total, 1 up, 2 in 2024-09-21T12:20:36.538 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:36 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T12:20:36.538 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:36 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:36.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:36 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/484522784' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "71d611f1-03c6-471c-baa2-602d6f2bc697"}]: dispatch 2024-09-21T12:20:36.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:36 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/484522784' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "71d611f1-03c6-471c-baa2-602d6f2bc697"}]': finished 2024-09-21T12:20:36.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:36 smithi148 ceph-mon[25818]: osdmap e11: 2 total, 1 up, 2 in 2024-09-21T12:20:36.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:36 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T12:20:36.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:36 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:37.381 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:37 smithi078 ceph-mon[23681]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:37.381 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:37 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/1305779332' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T12:20:37.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:37 smithi148 ceph-mon[25818]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:37.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:37 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/1305779332' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T12:20:39.243 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:39 smithi078 ceph-mon[23681]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:39.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:39 smithi148 ceph-mon[25818]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:41.466 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:41 smithi078 ceph-mon[23681]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:41.467 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:41 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:41.467 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:41 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:20:41.467 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:41 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:41.467 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:41 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:41.467 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:41 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:20:41.467 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:41 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:20:41.467 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:41 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:20:41.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:41 smithi148 ceph-mon[25818]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:41.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:41 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:41.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:41 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:20:41.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:41 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:41.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:41 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:41.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:41 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:20:41.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:41 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:20:41.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:41 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:20:42.331 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:42 smithi078 ceph-mon[23681]: Detected new or changed devices on smithi078 2024-09-21T12:20:42.331 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:42 smithi078 ceph-mon[23681]: Adjusting osd_memory_target on smithi078 to 12864M 2024-09-21T12:20:42.481 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:42 smithi148 ceph-mon[25818]: Detected new or changed devices on smithi078 2024-09-21T12:20:42.481 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:42 smithi148 ceph-mon[25818]: Adjusting osd_memory_target on smithi078 to 12864M 2024-09-21T12:20:44.041 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:43 smithi148 ceph-mon[25818]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:44.042 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:43 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:44.042 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:43 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-21T12:20:44.042 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:43 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:20:44.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:43 smithi078 ceph-mon[23681]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:44.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:43 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:44.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:43 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-21T12:20:44.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:43 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:20:45.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:44 smithi148 ceph-mon[25818]: Deploying daemon osd.1 on smithi078 2024-09-21T12:20:45.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:44 smithi078 ceph-mon[23681]: Deploying daemon osd.1 on smithi078 2024-09-21T12:20:46.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:45 smithi078 ceph-mon[23681]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:46.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:45 smithi148 ceph-mon[25818]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:48.050 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:47 smithi078 ceph-mon[23681]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:48.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:47 smithi148 ceph-mon[25818]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:49.940 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:49 smithi078 ceph-mon[23681]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:49.940 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:49 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:49.940 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:49 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:50.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:49 smithi148 ceph-mon[25818]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:50.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:49 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:50.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:49 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:51.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:51 smithi148 ceph-mon[25818]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:51.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:51 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:51.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:51 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:20:51.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:51 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:20:51.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:51 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:20:51.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:51 smithi078 ceph-mon[23681]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:51.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:51 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:51.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:51 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:20:51.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:51 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:20:51.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:51 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:20:53.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:53 smithi148 ceph-mon[25818]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:53.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:53 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:53.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:53 smithi078 ceph-mon[23681]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:53.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:53 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:53.677 INFO:teuthology.orchestra.run.smithi078.stdout:Created osd(s) 1 on host 'smithi078' 2024-09-21T12:20:54.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:54 smithi078 ceph-mon[23681]: from='osd.1 [v2:172.21.15.78:6810/2942847307,v1:172.21.15.78:6811/2942847307]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T12:20:54.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:54 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:54.346 DEBUG:teuthology.orchestra.run.smithi078:osd.1> sudo journalctl -f -n 0 -u ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.1.service 2024-09-21T12:20:54.348 INFO:tasks.cephadm:Deploying osd.2 on smithi078 with /dev/vg_nvme/lv_2... 2024-09-21T12:20:54.348 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_2 2024-09-21T12:20:54.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:54 smithi148 ceph-mon[25818]: from='osd.1 [v2:172.21.15.78:6810/2942847307,v1:172.21.15.78:6811/2942847307]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T12:20:54.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:54 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:20:55.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:55 smithi148 ceph-mon[25818]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:55.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:55 smithi148 ceph-mon[25818]: from='osd.1 [v2:172.21.15.78:6810/2942847307,v1:172.21.15.78:6811/2942847307]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T12:20:55.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:55 smithi148 ceph-mon[25818]: osdmap e12: 2 total, 1 up, 2 in 2024-09-21T12:20:55.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:55 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T12:20:55.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:55 smithi148 ceph-mon[25818]: from='osd.1 [v2:172.21.15.78:6810/2942847307,v1:172.21.15.78:6811/2942847307]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-09-21T12:20:55.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:55 smithi078 ceph-mon[23681]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:55.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:55 smithi078 ceph-mon[23681]: from='osd.1 [v2:172.21.15.78:6810/2942847307,v1:172.21.15.78:6811/2942847307]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T12:20:55.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:55 smithi078 ceph-mon[23681]: osdmap e12: 2 total, 1 up, 2 in 2024-09-21T12:20:55.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:55 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T12:20:55.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:55 smithi078 ceph-mon[23681]: from='osd.1 [v2:172.21.15.78:6810/2942847307,v1:172.21.15.78:6811/2942847307]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-09-21T12:20:55.557 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:20:55 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1[38191]: 2024-09-21T12:20:55.277+0000 7f6bf9d08700 -1 osd.1 0 waiting for initial osdmap 2024-09-21T12:20:55.557 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:20:55 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1[38191]: 2024-09-21T12:20:55.280+0000 7f6bf3e9e700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T12:20:56.528 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:56 smithi078 ceph-mon[23681]: from='osd.1 [v2:172.21.15.78:6810/2942847307,v1:172.21.15.78:6811/2942847307]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-09-21T12:20:56.528 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:56 smithi078 ceph-mon[23681]: osdmap e13: 2 total, 1 up, 2 in 2024-09-21T12:20:56.529 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:56 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T12:20:56.529 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:56 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T12:20:56.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:56 smithi148 ceph-mon[25818]: from='osd.1 [v2:172.21.15.78:6810/2942847307,v1:172.21.15.78:6811/2942847307]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-09-21T12:20:56.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:56 smithi148 ceph-mon[25818]: osdmap e13: 2 total, 1 up, 2 in 2024-09-21T12:20:56.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:56 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T12:20:56.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:56 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T12:20:57.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:57 smithi148 ceph-mon[25818]: purged_snaps scrub starts 2024-09-21T12:20:57.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:57 smithi148 ceph-mon[25818]: purged_snaps scrub ok 2024-09-21T12:20:57.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:57 smithi148 ceph-mon[25818]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:57.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:57 smithi148 ceph-mon[25818]: osd.1 [v2:172.21.15.78:6810/2942847307,v1:172.21.15.78:6811/2942847307] boot 2024-09-21T12:20:57.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:57 smithi148 ceph-mon[25818]: osdmap e14: 2 total, 2 up, 2 in 2024-09-21T12:20:57.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:57 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T12:20:57.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:57 smithi078 ceph-mon[23681]: purged_snaps scrub starts 2024-09-21T12:20:57.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:57 smithi078 ceph-mon[23681]: purged_snaps scrub ok 2024-09-21T12:20:57.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:57 smithi078 ceph-mon[23681]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-21T12:20:57.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:57 smithi078 ceph-mon[23681]: osd.1 [v2:172.21.15.78:6810/2942847307,v1:172.21.15.78:6811/2942847307] boot 2024-09-21T12:20:57.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:57 smithi078 ceph-mon[23681]: osdmap e14: 2 total, 2 up, 2 in 2024-09-21T12:20:57.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:57 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T12:20:58.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:58 smithi148 ceph-mon[25818]: osdmap e15: 2 total, 2 up, 2 in 2024-09-21T12:20:58.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:58 smithi078 ceph-mon[23681]: osdmap e15: 2 total, 2 up, 2 in 2024-09-21T12:20:58.887 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T12:20:58.905 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi078:vg_nvme/lv_2 2024-09-21T12:20:59.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:20:59 smithi148 ceph-mon[25818]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-21T12:20:59.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:20:59 smithi078 ceph-mon[23681]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-21T12:21:01.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:01 smithi078 ceph-mon[23681]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-21T12:21:01.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:01 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:01.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:01 smithi148 ceph-mon[25818]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-21T12:21:01.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:01 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:02.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:02 smithi148 ceph-mon[25818]: from='client.14272 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:21:02.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:02 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T12:21:02.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:02 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T12:21:02.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:02 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:21:02.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:02 smithi078 ceph-mon[23681]: from='client.14272 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:21:02.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:02 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T12:21:02.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:02 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T12:21:02.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:02 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:21:03.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:03 smithi148 ceph-mon[25818]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T12:21:03.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:03 smithi078 ceph-mon[23681]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T12:21:05.688 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:05 smithi078 ceph-mon[23681]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T12:21:05.688 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:05 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:05.688 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:05 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:21:05.688 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:05 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:21:05.688 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:05 smithi078 ceph-mon[23681]: Adjusting osd_memory_target on smithi078 to 6432M 2024-09-21T12:21:05.688 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:05 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:05.688 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:05 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:05.689 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:05 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:21:05.689 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:05 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:21:05.689 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:05 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:21:06.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:05 smithi148 ceph-mon[25818]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T12:21:06.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:05 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:06.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:05 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:21:06.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:05 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:21:06.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:05 smithi148 ceph-mon[25818]: Adjusting osd_memory_target on smithi078 to 6432M 2024-09-21T12:21:06.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:05 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:06.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:05 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:06.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:05 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:21:06.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:05 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:21:06.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:05 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:21:07.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:06 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/3575355541' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c200aece-da2c-4ccb-8dd7-f73c40b5b5e8"}]: dispatch 2024-09-21T12:21:07.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:06 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/3575355541' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c200aece-da2c-4ccb-8dd7-f73c40b5b5e8"}]': finished 2024-09-21T12:21:07.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:06 smithi148 ceph-mon[25818]: osdmap e16: 3 total, 2 up, 3 in 2024-09-21T12:21:07.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:06 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T12:21:07.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:06 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/3931348792' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T12:21:07.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:06 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/3575355541' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c200aece-da2c-4ccb-8dd7-f73c40b5b5e8"}]: dispatch 2024-09-21T12:21:07.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:06 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/3575355541' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c200aece-da2c-4ccb-8dd7-f73c40b5b5e8"}]': finished 2024-09-21T12:21:07.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:06 smithi078 ceph-mon[23681]: osdmap e16: 3 total, 2 up, 3 in 2024-09-21T12:21:07.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:06 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T12:21:07.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:06 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/3931348792' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T12:21:07.946 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:07 smithi078 ceph-mon[23681]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T12:21:08.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:07 smithi148 ceph-mon[25818]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T12:21:09.950 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:09 smithi078 ceph-mon[23681]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T12:21:10.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:09 smithi148 ceph-mon[25818]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T12:21:10.988 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:10 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:10.988 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:10 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:11.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:10 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:11.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:10 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:12.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:11 smithi148 ceph-mon[25818]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T12:21:12.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:11 smithi078 ceph-mon[23681]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T12:21:13.131 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:13 smithi078 ceph-mon[23681]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T12:21:13.131 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:13 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-21T12:21:13.131 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:13 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:21:13.131 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:13 smithi078 ceph-mon[23681]: Deploying daemon osd.2 on smithi078 2024-09-21T12:21:13.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:13 smithi148 ceph-mon[25818]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T12:21:13.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:13 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-21T12:21:13.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:13 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:21:13.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:13 smithi148 ceph-mon[25818]: Deploying daemon osd.2 on smithi078 2024-09-21T12:21:14.424 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:14 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/mirror_snapshot_schedule"}]: dispatch 2024-09-21T12:21:14.424 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:14 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/trash_purge_schedule"}]: dispatch 2024-09-21T12:21:14.444 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/mirror_snapshot_schedule"}]: dispatch 2024-09-21T12:21:14.444 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/trash_purge_schedule"}]: dispatch 2024-09-21T12:21:15.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:15 smithi078 ceph-mon[23681]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T12:21:15.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:15 smithi148 ceph-mon[25818]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T12:21:17.341 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:17 smithi078 ceph-mon[23681]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T12:21:17.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:17 smithi148 ceph-mon[25818]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T12:21:19.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:18 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:19.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:18 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:21:19.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:18 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:21:19.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:18 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:21:19.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:18 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:19.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:18 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:21:19.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:18 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:21:19.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:18 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:21:20.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:19 smithi148 ceph-mon[25818]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T12:21:20.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:19 smithi078 ceph-mon[23681]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T12:21:20.932 INFO:teuthology.orchestra.run.smithi078.stdout:Created osd(s) 2 on host 'smithi078' 2024-09-21T12:21:21.561 DEBUG:teuthology.orchestra.run.smithi078:osd.2> sudo journalctl -f -n 0 -u ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.2.service 2024-09-21T12:21:21.563 INFO:tasks.cephadm:Deploying osd.3 on smithi078 with /dev/vg_nvme/lv_1... 2024-09-21T12:21:21.563 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_1 2024-09-21T12:21:21.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:21 smithi078 ceph-mon[23681]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T12:21:21.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:21 smithi078 ceph-mon[23681]: from='osd.2 [v2:172.21.15.78:6818/1377778238,v1:172.21.15.78:6819/1377778238]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T12:21:21.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:21 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:22.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:21 smithi148 ceph-mon[25818]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T12:21:22.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:21 smithi148 ceph-mon[25818]: from='osd.2 [v2:172.21.15.78:6818/1377778238,v1:172.21.15.78:6819/1377778238]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T12:21:22.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:21 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:23.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:22 smithi148 ceph-mon[25818]: from='osd.2 [v2:172.21.15.78:6818/1377778238,v1:172.21.15.78:6819/1377778238]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T12:21:23.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:22 smithi148 ceph-mon[25818]: osdmap e17: 3 total, 2 up, 3 in 2024-09-21T12:21:23.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:22 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T12:21:23.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:22 smithi148 ceph-mon[25818]: from='osd.2 [v2:172.21.15.78:6818/1377778238,v1:172.21.15.78:6819/1377778238]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-09-21T12:21:23.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:22 smithi078 ceph-mon[23681]: from='osd.2 [v2:172.21.15.78:6818/1377778238,v1:172.21.15.78:6819/1377778238]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T12:21:23.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:22 smithi078 ceph-mon[23681]: osdmap e17: 3 total, 2 up, 3 in 2024-09-21T12:21:23.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:22 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T12:21:23.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:22 smithi078 ceph-mon[23681]: from='osd.2 [v2:172.21.15.78:6818/1377778238,v1:172.21.15.78:6819/1377778238]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-09-21T12:21:23.057 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:21:22 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2[41869]: 2024-09-21T12:21:22.732+0000 7fe572aaa700 -1 osd.2 0 waiting for initial osdmap 2024-09-21T12:21:23.057 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:21:22 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2[41869]: 2024-09-21T12:21:22.740+0000 7fe56b43d700 -1 osd.2 18 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T12:21:24.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:23 smithi148 ceph-mon[25818]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T12:21:24.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:23 smithi148 ceph-mon[25818]: from='osd.2 [v2:172.21.15.78:6818/1377778238,v1:172.21.15.78:6819/1377778238]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-09-21T12:21:24.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:23 smithi148 ceph-mon[25818]: osdmap e18: 3 total, 2 up, 3 in 2024-09-21T12:21:24.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:23 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T12:21:24.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:23 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T12:21:24.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:23 smithi078 ceph-mon[23681]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T12:21:24.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:23 smithi078 ceph-mon[23681]: from='osd.2 [v2:172.21.15.78:6818/1377778238,v1:172.21.15.78:6819/1377778238]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-09-21T12:21:24.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:23 smithi078 ceph-mon[23681]: osdmap e18: 3 total, 2 up, 3 in 2024-09-21T12:21:24.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:23 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T12:21:24.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:23 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T12:21:24.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:24 smithi078 ceph-mon[23681]: purged_snaps scrub starts 2024-09-21T12:21:24.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:24 smithi078 ceph-mon[23681]: purged_snaps scrub ok 2024-09-21T12:21:24.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:24 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T12:21:24.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:24 smithi078 ceph-mon[23681]: osd.2 [v2:172.21.15.78:6818/1377778238,v1:172.21.15.78:6819/1377778238] boot 2024-09-21T12:21:24.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:24 smithi078 ceph-mon[23681]: osdmap e19: 3 total, 3 up, 3 in 2024-09-21T12:21:24.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:24 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T12:21:24.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:24 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-21T12:21:24.854 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T12:21:24.873 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi078:vg_nvme/lv_1 2024-09-21T12:21:25.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:24 smithi148 ceph-mon[25818]: purged_snaps scrub starts 2024-09-21T12:21:25.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:24 smithi148 ceph-mon[25818]: purged_snaps scrub ok 2024-09-21T12:21:25.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:24 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T12:21:25.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:24 smithi148 ceph-mon[25818]: osd.2 [v2:172.21.15.78:6818/1377778238,v1:172.21.15.78:6819/1377778238] boot 2024-09-21T12:21:25.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:24 smithi148 ceph-mon[25818]: osdmap e19: 3 total, 3 up, 3 in 2024-09-21T12:21:25.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:24 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T12:21:25.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:24 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-21T12:21:26.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:25 smithi148 ceph-mon[25818]: pgmap v72: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T12:21:26.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:25 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-21T12:21:26.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:25 smithi148 ceph-mon[25818]: osdmap e20: 3 total, 3 up, 3 in 2024-09-21T12:21:26.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:25 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T12:21:26.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:25 smithi078 ceph-mon[23681]: pgmap v72: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-21T12:21:26.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:25 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-21T12:21:26.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:25 smithi078 ceph-mon[23681]: osdmap e20: 3 total, 3 up, 3 in 2024-09-21T12:21:26.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:25 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T12:21:26.924 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:26 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-21T12:21:26.925 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:26 smithi078 ceph-mon[23681]: osdmap e21: 3 total, 3 up, 3 in 2024-09-21T12:21:26.925 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:21:26 smithi078 sudo[43137]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T12:21:26.925 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:21:26 smithi078 sudo[43137]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T12:21:26.925 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:21:26 smithi078 sudo[43137]: pam_unix(sudo:session): session closed for user root 2024-09-21T12:21:26.925 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:21:26 smithi078 sudo[43164]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T12:21:27.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:26 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-21T12:21:27.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:26 smithi148 ceph-mon[25818]: osdmap e21: 3 total, 3 up, 3 in 2024-09-21T12:21:27.195 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:21:26 smithi078 sudo[43164]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T12:21:27.195 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:21:26 smithi078 sudo[43164]: pam_unix(sudo:session): session closed for user root 2024-09-21T12:21:27.467 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:21:27 smithi078 sudo[43189]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T12:21:27.467 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:21:27 smithi078 sudo[43189]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T12:21:27.467 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:21:27 smithi078 sudo[43189]: pam_unix(sudo:session): session closed for user root 2024-09-21T12:21:27.732 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:21:27 smithi078 sudo[43242]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T12:21:27.732 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:21:27 smithi078 sudo[43242]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T12:21:27.732 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:21:27 smithi078 sudo[43242]: pam_unix(sudo:session): session closed for user root 2024-09-21T12:21:27.993 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:21:27 smithi078 sudo[43314]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T12:21:27.993 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:21:27 smithi078 sudo[43314]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T12:21:27.993 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:21:27 smithi078 sudo[43314]: pam_unix(sudo:session): session closed for user root 2024-09-21T12:21:27.994 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:27 smithi078 ceph-mon[23681]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-21T12:21:27.994 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:27 smithi078 ceph-mon[23681]: osdmap e22: 3 total, 3 up, 3 in 2024-09-21T12:21:27.994 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:27 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:27.994 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:27 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T12:21:27.994 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:27 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T12:21:27.994 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:27 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:21:28.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:27 smithi148 ceph-mon[25818]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-21T12:21:28.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:27 smithi148 ceph-mon[25818]: osdmap e22: 3 total, 3 up, 3 in 2024-09-21T12:21:28.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:27 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:28.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:27 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T12:21:28.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:27 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T12:21:28.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:27 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:21:28.245 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:21:27 smithi078 sudo[43323]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T12:21:28.245 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:21:27 smithi078 sudo[43323]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T12:21:28.245 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:21:27 smithi078 sudo[43323]: pam_unix(sudo:session): session closed for user root 2024-09-21T12:21:28.505 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:28 smithi078 sudo[43351]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T12:21:28.505 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:28 smithi078 sudo[43351]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T12:21:28.505 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:28 smithi078 sudo[43351]: pam_unix(sudo:session): session closed for user root 2024-09-21T12:21:28.506 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:28 smithi078 sudo[43358]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T12:21:28.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:28 smithi078 sudo[43358]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T12:21:28.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:28 smithi078 sudo[43358]: pam_unix(sudo:session): session closed for user root 2024-09-21T12:21:29.002 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:28 smithi148 sudo[28120]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T12:21:29.002 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:28 smithi148 sudo[28120]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T12:21:29.002 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:28 smithi148 sudo[28120]: pam_unix(sudo:session): session closed for user root 2024-09-21T12:21:29.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:29 smithi148 sudo[28123]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T12:21:29.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:29 smithi148 sudo[28123]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-21T12:21:29.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:29 smithi148 sudo[28123]: pam_unix(sudo:session): session closed for user root 2024-09-21T12:21:29.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:29 smithi148 ceph-mon[25818]: from='client.14294 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:21:29.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:29 smithi148 ceph-mon[25818]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T12:21:29.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:29 smithi148 ceph-mon[25818]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T12:21:29.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:29 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-21T12:21:29.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:29 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi148"}]: dispatch 2024-09-21T12:21:29.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:29 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-21T12:21:29.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:29 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi148"}]: dispatch 2024-09-21T12:21:29.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:29 smithi078 ceph-mon[23681]: from='client.14294 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:21:29.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:29 smithi078 ceph-mon[23681]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T12:21:29.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:29 smithi078 ceph-mon[23681]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T12:21:29.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:29 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-21T12:21:29.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:29 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi148"}]: dispatch 2024-09-21T12:21:29.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:29 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-21T12:21:29.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:29 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi148"}]: dispatch 2024-09-21T12:21:30.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:30 smithi148 ceph-mon[25818]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-09-21T12:21:30.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:30 smithi148 ceph-mon[25818]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T12:21:30.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:30 smithi148 ceph-mon[25818]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T12:21:30.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:30 smithi148 ceph-mon[25818]: mgrmap e19: smithi078.ntwyan(active, since 2m), standbys: smithi148.oyiqmx 2024-09-21T12:21:30.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:30 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:30.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:30 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:30.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:30 smithi078 ceph-mon[23681]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-09-21T12:21:30.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:30 smithi078 ceph-mon[23681]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T12:21:30.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:30 smithi078 ceph-mon[23681]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T12:21:30.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:30 smithi078 ceph-mon[23681]: mgrmap e19: smithi078.ntwyan(active, since 2m), standbys: smithi148.oyiqmx 2024-09-21T12:21:30.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:30 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:30.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:30 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:31.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:31 smithi148 ceph-mon[25818]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-09-21T12:21:31.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:31 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/1861786981' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7db813e7-9c1a-4f95-b4b7-c871a878cf15"}]: dispatch 2024-09-21T12:21:31.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:31 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/1861786981' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7db813e7-9c1a-4f95-b4b7-c871a878cf15"}]': finished 2024-09-21T12:21:31.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:31 smithi148 ceph-mon[25818]: osdmap e23: 4 total, 3 up, 4 in 2024-09-21T12:21:31.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:31 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T12:21:31.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:31 smithi078 ceph-mon[23681]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-09-21T12:21:31.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:31 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/1861786981' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7db813e7-9c1a-4f95-b4b7-c871a878cf15"}]: dispatch 2024-09-21T12:21:31.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:31 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/1861786981' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7db813e7-9c1a-4f95-b4b7-c871a878cf15"}]': finished 2024-09-21T12:21:31.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:31 smithi078 ceph-mon[23681]: osdmap e23: 4 total, 3 up, 4 in 2024-09-21T12:21:31.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:31 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T12:21:32.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:32 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/1136661706' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T12:21:32.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:32 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/1136661706' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T12:21:33.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:33 smithi078 ceph-mon[23681]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-21T12:21:33.444 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:33 smithi148 ceph-mon[25818]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-21T12:21:35.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:35 smithi078 ceph-mon[23681]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-21T12:21:35.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:35 smithi148 ceph-mon[25818]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-21T12:21:37.254 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:36 smithi078 ceph-mon[23681]: Detected new or changed devices on smithi078 2024-09-21T12:21:37.254 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:36 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:37.254 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:36 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:21:37.254 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:36 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:21:37.254 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:36 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:21:37.255 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:36 smithi078 ceph-mon[23681]: Adjusting osd_memory_target on smithi078 to 4288M 2024-09-21T12:21:37.255 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:36 smithi078 ceph-mon[23681]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-21T12:21:37.255 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:36 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:37.255 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:36 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:37.255 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:36 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:21:37.255 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:36 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:21:37.255 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:36 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:21:37.255 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:36 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:37.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:36 smithi148 ceph-mon[25818]: Detected new or changed devices on smithi078 2024-09-21T12:21:37.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:36 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:37.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:36 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:21:37.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:36 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:21:37.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:36 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:21:37.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:36 smithi148 ceph-mon[25818]: Adjusting osd_memory_target on smithi078 to 4288M 2024-09-21T12:21:37.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:36 smithi148 ceph-mon[25818]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-21T12:21:37.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:36 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:37.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:36 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:37.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:36 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:21:37.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:36 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:21:37.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:36 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:21:37.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:36 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:38.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:38 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-21T12:21:38.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:38 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:21:38.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:38 smithi078 ceph-mon[23681]: Deploying daemon osd.3 on smithi078 2024-09-21T12:21:38.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:38 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-21T12:21:38.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:38 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:21:38.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:38 smithi148 ceph-mon[25818]: Deploying daemon osd.3 on smithi078 2024-09-21T12:21:39.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:39 smithi078 ceph-mon[23681]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-21T12:21:39.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:39 smithi148 ceph-mon[25818]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-21T12:21:41.333 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:41 smithi078 ceph-mon[23681]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-21T12:21:41.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:41 smithi148 ceph-mon[25818]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-21T12:21:43.141 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:43 smithi078 ceph-mon[23681]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 68 KiB/s wr, 4 op/s 2024-09-21T12:21:43.141 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:43 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:43.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:43 smithi148 ceph-mon[25818]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 68 KiB/s wr, 4 op/s 2024-09-21T12:21:43.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:43 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:44.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:44 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:21:44.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:44 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:21:44.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:44 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:21:44.459 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:44 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:21:44.459 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:44 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:21:44.459 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:44 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:21:45.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:45 smithi078 ceph-mon[23681]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-21T12:21:45.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:45 smithi148 ceph-mon[25818]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-21T12:21:45.856 INFO:teuthology.orchestra.run.smithi078.stdout:Created osd(s) 3 on host 'smithi078' 2024-09-21T12:21:46.502 DEBUG:teuthology.orchestra.run.smithi078:osd.3> sudo journalctl -f -n 0 -u ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.3.service 2024-09-21T12:21:46.504 INFO:tasks.cephadm:Deploying osd.4 on smithi148 with /dev/vg_nvme/lv_4... 2024-09-21T12:21:46.504 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_4 2024-09-21T12:21:47.112 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:46 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:47.112 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:46 smithi148 ceph-mon[25818]: from='osd.3 [v2:172.21.15.78:6826/880260351,v1:172.21.15.78:6827/880260351]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T12:21:47.175 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:46 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:47.175 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:46 smithi078 ceph-mon[23681]: from='osd.3 [v2:172.21.15.78:6826/880260351,v1:172.21.15.78:6827/880260351]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T12:21:47.966 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:47 smithi148 ceph-mon[25818]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-21T12:21:47.966 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:47 smithi148 ceph-mon[25818]: from='osd.3 [v2:172.21.15.78:6826/880260351,v1:172.21.15.78:6827/880260351]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T12:21:47.966 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:47 smithi148 ceph-mon[25818]: osdmap e24: 4 total, 3 up, 4 in 2024-09-21T12:21:47.966 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:47 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T12:21:47.966 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:47 smithi148 ceph-mon[25818]: from='osd.3 [v2:172.21.15.78:6826/880260351,v1:172.21.15.78:6827/880260351]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-09-21T12:21:48.186 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:47 smithi078 ceph-mon[23681]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-21T12:21:48.186 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:47 smithi078 ceph-mon[23681]: from='osd.3 [v2:172.21.15.78:6826/880260351,v1:172.21.15.78:6827/880260351]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T12:21:48.186 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:47 smithi078 ceph-mon[23681]: osdmap e24: 4 total, 3 up, 4 in 2024-09-21T12:21:48.186 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:47 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T12:21:48.186 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:47 smithi078 ceph-mon[23681]: from='osd.3 [v2:172.21.15.78:6826/880260351,v1:172.21.15.78:6827/880260351]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-09-21T12:21:48.186 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:21:47 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3[45093]: 2024-09-21T12:21:47.862+0000 7f1c577f6700 -1 osd.3 0 waiting for initial osdmap 2024-09-21T12:21:48.187 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:21:47 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3[45093]: 2024-09-21T12:21:47.868+0000 7f1c5218d700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T12:21:49.113 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:48 smithi148 ceph-mon[25818]: from='osd.3 [v2:172.21.15.78:6826/880260351,v1:172.21.15.78:6827/880260351]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-09-21T12:21:49.113 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:48 smithi148 ceph-mon[25818]: osdmap e25: 4 total, 3 up, 4 in 2024-09-21T12:21:49.113 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:48 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T12:21:49.113 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:48 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T12:21:49.118 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-09-21T12:21:49.131 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:48 smithi078 ceph-mon[23681]: from='osd.3 [v2:172.21.15.78:6826/880260351,v1:172.21.15.78:6827/880260351]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-09-21T12:21:49.131 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:48 smithi078 ceph-mon[23681]: osdmap e25: 4 total, 3 up, 4 in 2024-09-21T12:21:49.131 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:48 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T12:21:49.131 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:48 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T12:21:49.138 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi148:vg_nvme/lv_4 2024-09-21T12:21:49.910 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:49 smithi078 ceph-mon[23681]: purged_snaps scrub starts 2024-09-21T12:21:49.910 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:49 smithi078 ceph-mon[23681]: purged_snaps scrub ok 2024-09-21T12:21:49.910 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:49 smithi078 ceph-mon[23681]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-21T12:21:49.911 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:49 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T12:21:49.911 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:49 smithi078 ceph-mon[23681]: osd.3 [v2:172.21.15.78:6826/880260351,v1:172.21.15.78:6827/880260351] boot 2024-09-21T12:21:49.911 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:49 smithi078 ceph-mon[23681]: osdmap e26: 4 total, 4 up, 4 in 2024-09-21T12:21:49.911 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:49 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T12:21:50.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:49 smithi148 ceph-mon[25818]: purged_snaps scrub starts 2024-09-21T12:21:50.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:49 smithi148 ceph-mon[25818]: purged_snaps scrub ok 2024-09-21T12:21:50.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:49 smithi148 ceph-mon[25818]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-21T12:21:50.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:49 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T12:21:50.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:49 smithi148 ceph-mon[25818]: osd.3 [v2:172.21.15.78:6826/880260351,v1:172.21.15.78:6827/880260351] boot 2024-09-21T12:21:50.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:49 smithi148 ceph-mon[25818]: osdmap e26: 4 total, 4 up, 4 in 2024-09-21T12:21:50.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:49 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T12:21:51.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:50 smithi148 ceph-mon[25818]: osdmap e27: 4 total, 4 up, 4 in 2024-09-21T12:21:51.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:50 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:51.173 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:50 smithi078 ceph-mon[23681]: osdmap e27: 4 total, 4 up, 4 in 2024-09-21T12:21:51.173 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:50 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:52.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:51 smithi148 ceph-mon[25818]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T12:21:52.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:51 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T12:21:52.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:51 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T12:21:52.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:51 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:21:52.222 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:51 smithi078 ceph-mon[23681]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T12:21:52.222 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:51 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T12:21:52.222 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:51 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T12:21:52.222 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:51 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:21:53.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:52 smithi148 ceph-mon[25818]: from='client.24133 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi148:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:21:53.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:52 smithi078 ceph-mon[23681]: from='client.24133 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi148:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:21:54.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:53 smithi078 ceph-mon[23681]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T12:21:54.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:53 smithi078 ceph-mon[23681]: from='client.? 172.21.15.148:0/2084050978' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2c7f0c21-a23f-4039-9a82-8cd307394520"}]: dispatch 2024-09-21T12:21:54.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:53 smithi078 ceph-mon[23681]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2c7f0c21-a23f-4039-9a82-8cd307394520"}]: dispatch 2024-09-21T12:21:54.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:53 smithi078 ceph-mon[23681]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2c7f0c21-a23f-4039-9a82-8cd307394520"}]': finished 2024-09-21T12:21:54.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:53 smithi078 ceph-mon[23681]: osdmap e28: 5 total, 4 up, 5 in 2024-09-21T12:21:54.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:53 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T12:21:54.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:53 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:54.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:53 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:21:54.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:53 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:21:54.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:53 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:21:54.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:53 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:21:54.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:53 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:54.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:53 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:54.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:53 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:21:54.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:53 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:21:54.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:53 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:21:54.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:53 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:54.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:53 smithi148 ceph-mon[25818]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T12:21:54.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:53 smithi148 ceph-mon[25818]: from='client.? 172.21.15.148:0/2084050978' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2c7f0c21-a23f-4039-9a82-8cd307394520"}]: dispatch 2024-09-21T12:21:54.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:53 smithi148 ceph-mon[25818]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2c7f0c21-a23f-4039-9a82-8cd307394520"}]: dispatch 2024-09-21T12:21:54.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:53 smithi148 ceph-mon[25818]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2c7f0c21-a23f-4039-9a82-8cd307394520"}]': finished 2024-09-21T12:21:54.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:53 smithi148 ceph-mon[25818]: osdmap e28: 5 total, 4 up, 5 in 2024-09-21T12:21:54.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:53 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T12:21:54.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:53 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:54.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:53 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:21:54.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:53 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:21:54.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:53 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:21:54.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:53 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:21:54.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:53 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:54.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:53 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:54.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:53 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:21:54.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:53 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:21:54.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:53 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:21:54.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:53 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:21:55.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:54 smithi148 ceph-mon[25818]: Adjusting osd_memory_target on smithi078 to 3216M 2024-09-21T12:21:55.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:54 smithi148 ceph-mon[25818]: from='client.? 172.21.15.148:0/1942281317' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T12:21:55.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:54 smithi078 ceph-mon[23681]: Adjusting osd_memory_target on smithi078 to 3216M 2024-09-21T12:21:55.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:54 smithi078 ceph-mon[23681]: from='client.? 172.21.15.148:0/1942281317' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T12:21:56.154 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:55 smithi148 ceph-mon[25818]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T12:21:56.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:55 smithi078 ceph-mon[23681]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T12:21:58.197 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:57 smithi148 ceph-mon[25818]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T12:21:58.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:57 smithi078 ceph-mon[23681]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T12:22:00.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:59 smithi148 ceph-mon[25818]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T12:22:00.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:59 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-21T12:22:00.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:21:59 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:22:00.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:59 smithi078 ceph-mon[23681]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T12:22:00.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:59 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-21T12:22:00.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:21:59 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:22:00.939 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:00 smithi148 ceph-mon[25818]: Deploying daemon osd.4 on smithi148 2024-09-21T12:22:01.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:00 smithi078 ceph-mon[23681]: Deploying daemon osd.4 on smithi148 2024-09-21T12:22:02.216 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:01 smithi148 ceph-mon[25818]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T12:22:02.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:01 smithi078 ceph-mon[23681]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T12:22:04.047 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:03 smithi148 ceph-mon[25818]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T12:22:04.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:03 smithi078 ceph-mon[23681]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T12:22:05.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:05 smithi148 ceph-mon[25818]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T12:22:05.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:05 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:05.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:05 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:22:05.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:05 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:22:05.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:05 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:22:05.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:05 smithi078 ceph-mon[23681]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T12:22:05.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:05 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:05.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:05 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:22:05.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:05 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:22:05.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:05 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:22:07.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:07 smithi148 ceph-mon[25818]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T12:22:07.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:07 smithi078 ceph-mon[23681]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T12:22:08.131 INFO:teuthology.orchestra.run.smithi148.stdout:Created osd(s) 4 on host 'smithi148' 2024-09-21T12:22:08.436 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:08 smithi148 ceph-mon[25818]: from='osd.4 [v2:172.21.15.148:6800/3618711344,v1:172.21.15.148:6801/3618711344]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T12:22:08.436 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:08 smithi148 ceph-mon[25818]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T12:22:08.436 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:08 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:08.437 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:08 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:08.437 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:08 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:08.437 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:08 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:22:08.437 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:08 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:22:08.437 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:08 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:22:08.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:08 smithi078 ceph-mon[23681]: from='osd.4 [v2:172.21.15.148:6800/3618711344,v1:172.21.15.148:6801/3618711344]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T12:22:08.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:08 smithi078 ceph-mon[23681]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T12:22:08.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:08 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:08.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:08 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:08.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:08 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:08.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:08 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:22:08.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:08 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:22:08.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:08 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:22:08.725 DEBUG:teuthology.orchestra.run.smithi148:osd.4> sudo journalctl -f -n 0 -u ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.4.service 2024-09-21T12:22:08.728 INFO:tasks.cephadm:Deploying osd.5 on smithi148 with /dev/vg_nvme/lv_3... 2024-09-21T12:22:08.728 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_3 2024-09-21T12:22:09.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:09 smithi148 ceph-mon[25818]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T12:22:09.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:09 smithi148 ceph-mon[25818]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T12:22:09.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:09 smithi148 ceph-mon[25818]: osdmap e29: 5 total, 4 up, 5 in 2024-09-21T12:22:09.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:09 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T12:22:09.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:09 smithi148 ceph-mon[25818]: from='osd.4 [v2:172.21.15.148:6800/3618711344,v1:172.21.15.148:6801/3618711344]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-09-21T12:22:09.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:09 smithi148 ceph-mon[25818]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-09-21T12:22:09.551 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:22:09 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4[30019]: 2024-09-21T12:22:09.302+0000 7f5967e2d700 -1 osd.4 0 waiting for initial osdmap 2024-09-21T12:22:09.551 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:22:09 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4[30019]: 2024-09-21T12:22:09.306+0000 7f59627c4700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T12:22:09.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:09 smithi078 ceph-mon[23681]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T12:22:09.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:09 smithi078 ceph-mon[23681]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T12:22:09.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:09 smithi078 ceph-mon[23681]: osdmap e29: 5 total, 4 up, 5 in 2024-09-21T12:22:09.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:09 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T12:22:09.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:09 smithi078 ceph-mon[23681]: from='osd.4 [v2:172.21.15.148:6800/3618711344,v1:172.21.15.148:6801/3618711344]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-09-21T12:22:09.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:09 smithi078 ceph-mon[23681]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-09-21T12:22:10.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:10 smithi148 ceph-mon[25818]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]': finished 2024-09-21T12:22:10.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:10 smithi148 ceph-mon[25818]: osdmap e30: 5 total, 4 up, 5 in 2024-09-21T12:22:10.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:10 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T12:22:10.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:10 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T12:22:10.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:10 smithi078 ceph-mon[23681]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]': finished 2024-09-21T12:22:10.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:10 smithi078 ceph-mon[23681]: osdmap e30: 5 total, 4 up, 5 in 2024-09-21T12:22:10.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:10 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T12:22:10.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:10 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T12:22:11.728 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:11 smithi148 ceph-mon[25818]: purged_snaps scrub starts 2024-09-21T12:22:11.729 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:11 smithi148 ceph-mon[25818]: purged_snaps scrub ok 2024-09-21T12:22:11.729 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:11 smithi148 ceph-mon[25818]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T12:22:11.729 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:11 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T12:22:11.729 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:11 smithi148 ceph-mon[25818]: osd.4 [v2:172.21.15.148:6800/3618711344,v1:172.21.15.148:6801/3618711344] boot 2024-09-21T12:22:11.729 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:11 smithi148 ceph-mon[25818]: osdmap e31: 5 total, 5 up, 5 in 2024-09-21T12:22:11.729 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:11 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T12:22:11.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:11 smithi078 ceph-mon[23681]: purged_snaps scrub starts 2024-09-21T12:22:11.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:11 smithi078 ceph-mon[23681]: purged_snaps scrub ok 2024-09-21T12:22:11.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:11 smithi078 ceph-mon[23681]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-21T12:22:11.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:11 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T12:22:11.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:11 smithi078 ceph-mon[23681]: osd.4 [v2:172.21.15.148:6800/3618711344,v1:172.21.15.148:6801/3618711344] boot 2024-09-21T12:22:11.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:11 smithi078 ceph-mon[23681]: osdmap e31: 5 total, 5 up, 5 in 2024-09-21T12:22:11.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:11 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T12:22:12.535 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:12 smithi148 ceph-mon[25818]: osdmap e32: 5 total, 5 up, 5 in 2024-09-21T12:22:12.535 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:12 smithi148 ceph-mon[25818]: Detected new or changed devices on smithi148 2024-09-21T12:22:12.535 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:12 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:12.536 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:12 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:22:12.536 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:12 smithi148 ceph-mon[25818]: Adjusting osd_memory_target on smithi148 to 15936M 2024-09-21T12:22:12.536 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:12 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:12.536 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:12 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:12.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:12 smithi078 ceph-mon[23681]: osdmap e32: 5 total, 5 up, 5 in 2024-09-21T12:22:12.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:12 smithi078 ceph-mon[23681]: Detected new or changed devices on smithi148 2024-09-21T12:22:12.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:12 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:12.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:12 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:22:12.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:12 smithi078 ceph-mon[23681]: Adjusting osd_memory_target on smithi148 to 15936M 2024-09-21T12:22:12.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:12 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:12.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:12 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:13.638 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:13 smithi148 ceph-mon[25818]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-21T12:22:13.638 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:13 smithi148 ceph-mon[25818]: osdmap e33: 5 total, 5 up, 5 in 2024-09-21T12:22:13.643 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-09-21T12:22:13.660 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi148:vg_nvme/lv_3 2024-09-21T12:22:13.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:13 smithi078 ceph-mon[23681]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-21T12:22:13.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:13 smithi078 ceph-mon[23681]: osdmap e33: 5 total, 5 up, 5 in 2024-09-21T12:22:14.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:14 smithi148 ceph-mon[25818]: osdmap e34: 5 total, 5 up, 5 in 2024-09-21T12:22:14.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:14 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/trash_purge_schedule"}]: dispatch 2024-09-21T12:22:14.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:14 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/mirror_snapshot_schedule"}]: dispatch 2024-09-21T12:22:14.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:14 smithi078 ceph-mon[23681]: osdmap e34: 5 total, 5 up, 5 in 2024-09-21T12:22:14.808 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/trash_purge_schedule"}]: dispatch 2024-09-21T12:22:14.808 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/mirror_snapshot_schedule"}]: dispatch 2024-09-21T12:22:15.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:15 smithi148 ceph-mon[25818]: pgmap v112: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-21T12:22:15.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:15 smithi148 ceph-mon[25818]: osdmap e35: 5 total, 5 up, 5 in 2024-09-21T12:22:15.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:15 smithi078 ceph-mon[23681]: pgmap v112: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-21T12:22:15.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:15 smithi078 ceph-mon[23681]: osdmap e35: 5 total, 5 up, 5 in 2024-09-21T12:22:16.511 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:16 smithi148 ceph-mon[25818]: from='client.24151 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi148:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:22:16.511 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:16 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T12:22:16.511 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:16 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T12:22:16.511 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:16 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:22:16.707 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:16 smithi078 ceph-mon[23681]: from='client.24151 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi148:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:22:16.707 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:16 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T12:22:16.707 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:16 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T12:22:16.707 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:16 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:22:17.723 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:17 smithi078 ceph-mon[23681]: pgmap v114: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T12:22:17.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:17 smithi148 ceph-mon[25818]: pgmap v114: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T12:22:18.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:18 smithi078 ceph-mon[23681]: from='client.? 172.21.15.148:0/4123112763' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8c2fb91b-c05c-4f2d-a272-d15a98be394d"}]: dispatch 2024-09-21T12:22:18.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:18 smithi078 ceph-mon[23681]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8c2fb91b-c05c-4f2d-a272-d15a98be394d"}]: dispatch 2024-09-21T12:22:18.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:18 smithi078 ceph-mon[23681]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8c2fb91b-c05c-4f2d-a272-d15a98be394d"}]': finished 2024-09-21T12:22:18.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:18 smithi078 ceph-mon[23681]: osdmap e36: 6 total, 5 up, 6 in 2024-09-21T12:22:18.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:18 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T12:22:18.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:18 smithi078 ceph-mon[23681]: from='client.? 172.21.15.148:0/3044715845' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T12:22:18.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:18 smithi148 ceph-mon[25818]: from='client.? 172.21.15.148:0/4123112763' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8c2fb91b-c05c-4f2d-a272-d15a98be394d"}]: dispatch 2024-09-21T12:22:18.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:18 smithi148 ceph-mon[25818]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8c2fb91b-c05c-4f2d-a272-d15a98be394d"}]: dispatch 2024-09-21T12:22:18.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:18 smithi148 ceph-mon[25818]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8c2fb91b-c05c-4f2d-a272-d15a98be394d"}]': finished 2024-09-21T12:22:18.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:18 smithi148 ceph-mon[25818]: osdmap e36: 6 total, 5 up, 6 in 2024-09-21T12:22:18.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:18 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T12:22:18.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:18 smithi148 ceph-mon[25818]: from='client.? 172.21.15.148:0/3044715845' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T12:22:20.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:19 smithi148 ceph-mon[25818]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-09-21T12:22:20.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:19 smithi078 ceph-mon[23681]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 91 KiB/s, 0 objects/s recovering 2024-09-21T12:22:21.873 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:21 smithi148 ceph-mon[25818]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-09-21T12:22:22.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:21 smithi078 ceph-mon[23681]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-09-21T12:22:23.977 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:23 smithi148 ceph-mon[25818]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-21T12:22:24.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:23 smithi078 ceph-mon[23681]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-21T12:22:25.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:24 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-21T12:22:25.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:24 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:22:25.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:24 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-21T12:22:25.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:24 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:22:25.899 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:25 smithi148 ceph-mon[25818]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-09-21T12:22:25.899 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:25 smithi148 ceph-mon[25818]: Deploying daemon osd.5 on smithi148 2024-09-21T12:22:26.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:25 smithi078 ceph-mon[23681]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-09-21T12:22:26.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:25 smithi078 ceph-mon[23681]: Deploying daemon osd.5 on smithi148 2024-09-21T12:22:28.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:27 smithi148 ceph-mon[25818]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-21T12:22:28.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:27 smithi078 ceph-mon[23681]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-21T12:22:29.991 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:29 smithi148 ceph-mon[25818]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T12:22:29.991 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:29 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:29.991 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:29 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:22:29.991 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:29 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:22:29.991 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:29 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:22:30.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:29 smithi078 ceph-mon[23681]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T12:22:30.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:29 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:30.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:29 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:22:30.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:29 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:22:30.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:29 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:22:32.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:31 smithi148 ceph-mon[25818]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T12:22:32.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:31 smithi078 ceph-mon[23681]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T12:22:32.770 INFO:teuthology.orchestra.run.smithi148.stdout:Created osd(s) 5 on host 'smithi148' 2024-09-21T12:22:33.350 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:33 smithi148 ceph-mon[25818]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T12:22:33.350 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:33 smithi148 ceph-mon[25818]: from='osd.5 [v2:172.21.15.148:6808/1612483000,v1:172.21.15.148:6809/1612483000]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T12:22:33.351 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:33 smithi148 ceph-mon[25818]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T12:22:33.351 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:33 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:33.374 DEBUG:teuthology.orchestra.run.smithi148:osd.5> sudo journalctl -f -n 0 -u ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.5.service 2024-09-21T12:22:33.377 INFO:tasks.cephadm:Deploying osd.6 on smithi148 with /dev/vg_nvme/lv_2... 2024-09-21T12:22:33.377 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_2 2024-09-21T12:22:33.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:33 smithi078 ceph-mon[23681]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T12:22:33.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:33 smithi078 ceph-mon[23681]: from='osd.5 [v2:172.21.15.148:6808/1612483000,v1:172.21.15.148:6809/1612483000]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T12:22:33.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:33 smithi078 ceph-mon[23681]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T12:22:33.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:33 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:34.244 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:34 smithi148 ceph-mon[25818]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T12:22:34.245 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:34 smithi148 ceph-mon[25818]: osdmap e37: 6 total, 5 up, 6 in 2024-09-21T12:22:34.245 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:34 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T12:22:34.245 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:34 smithi148 ceph-mon[25818]: from='osd.5 [v2:172.21.15.148:6808/1612483000,v1:172.21.15.148:6809/1612483000]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-09-21T12:22:34.245 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:34 smithi148 ceph-mon[25818]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-09-21T12:22:34.245 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:34 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:34.245 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:22:34 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[32971]: 2024-09-21T12:22:34.101+0000 7fb53e29a700 -1 osd.5 0 waiting for initial osdmap 2024-09-21T12:22:34.245 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:22:34 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[32971]: 2024-09-21T12:22:34.106+0000 7fb53ac35700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T12:22:34.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:34 smithi078 ceph-mon[23681]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T12:22:34.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:34 smithi078 ceph-mon[23681]: osdmap e37: 6 total, 5 up, 6 in 2024-09-21T12:22:34.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:34 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T12:22:34.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:34 smithi078 ceph-mon[23681]: from='osd.5 [v2:172.21.15.148:6808/1612483000,v1:172.21.15.148:6809/1612483000]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-09-21T12:22:34.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:34 smithi078 ceph-mon[23681]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-09-21T12:22:34.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:34 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:35.239 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:35 smithi148 ceph-mon[25818]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T12:22:35.239 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:35 smithi148 ceph-mon[25818]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]': finished 2024-09-21T12:22:35.239 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:35 smithi148 ceph-mon[25818]: osdmap e38: 6 total, 5 up, 6 in 2024-09-21T12:22:35.239 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:35 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T12:22:35.239 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:35 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T12:22:35.558 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:35 smithi078 ceph-mon[23681]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T12:22:35.558 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:35 smithi078 ceph-mon[23681]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]': finished 2024-09-21T12:22:35.558 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:35 smithi078 ceph-mon[23681]: osdmap e38: 6 total, 5 up, 6 in 2024-09-21T12:22:35.558 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:35 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T12:22:35.558 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:35 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T12:22:36.263 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:36 smithi148 ceph-mon[25818]: purged_snaps scrub starts 2024-09-21T12:22:36.263 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:36 smithi148 ceph-mon[25818]: purged_snaps scrub ok 2024-09-21T12:22:36.264 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:36 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T12:22:36.264 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:36 smithi148 ceph-mon[25818]: osd.5 [v2:172.21.15.148:6808/1612483000,v1:172.21.15.148:6809/1612483000] boot 2024-09-21T12:22:36.264 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:36 smithi148 ceph-mon[25818]: osdmap e39: 6 total, 6 up, 6 in 2024-09-21T12:22:36.264 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:36 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T12:22:36.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:36 smithi078 ceph-mon[23681]: purged_snaps scrub starts 2024-09-21T12:22:36.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:36 smithi078 ceph-mon[23681]: purged_snaps scrub ok 2024-09-21T12:22:36.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:36 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T12:22:36.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:36 smithi078 ceph-mon[23681]: osd.5 [v2:172.21.15.148:6808/1612483000,v1:172.21.15.148:6809/1612483000] boot 2024-09-21T12:22:36.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:36 smithi078 ceph-mon[23681]: osdmap e39: 6 total, 6 up, 6 in 2024-09-21T12:22:36.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:36 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T12:22:37.034 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-09-21T12:22:37.055 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi148:vg_nvme/lv_2 2024-09-21T12:22:37.295 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:37 smithi148 ceph-mon[25818]: pgmap v128: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T12:22:37.295 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:37 smithi148 ceph-mon[25818]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T12:22:37.295 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:37 smithi148 ceph-mon[25818]: osdmap e40: 6 total, 6 up, 6 in 2024-09-21T12:22:37.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:37 smithi078 ceph-mon[23681]: pgmap v128: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-21T12:22:37.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:37 smithi078 ceph-mon[23681]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T12:22:37.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:37 smithi078 ceph-mon[23681]: osdmap e40: 6 total, 6 up, 6 in 2024-09-21T12:22:38.229 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:38 smithi148 ceph-mon[25818]: osdmap e41: 6 total, 6 up, 6 in 2024-09-21T12:22:38.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:38 smithi078 ceph-mon[23681]: osdmap e41: 6 total, 6 up, 6 in 2024-09-21T12:22:39.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:39 smithi148 ceph-mon[25818]: pgmap v131: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-21T12:22:39.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:39 smithi148 ceph-mon[25818]: osdmap e42: 6 total, 6 up, 6 in 2024-09-21T12:22:39.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:39 smithi078 ceph-mon[23681]: pgmap v131: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-21T12:22:39.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:39 smithi078 ceph-mon[23681]: osdmap e42: 6 total, 6 up, 6 in 2024-09-21T12:22:40.299 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:40 smithi148 ceph-mon[25818]: osdmap e43: 6 total, 6 up, 6 in 2024-09-21T12:22:40.299 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:40 smithi148 ceph-mon[25818]: Detected new or changed devices on smithi148 2024-09-21T12:22:40.299 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:40 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:40.299 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:40 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:22:40.299 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:40 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:22:40.299 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:40 smithi148 ceph-mon[25818]: Adjusting osd_memory_target on smithi148 to 7968M 2024-09-21T12:22:40.299 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:40 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:40.299 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:40 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:40.299 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:40 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:22:40.299 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:40 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:22:40.299 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:40 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:22:40.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:40 smithi078 ceph-mon[23681]: osdmap e43: 6 total, 6 up, 6 in 2024-09-21T12:22:40.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:40 smithi078 ceph-mon[23681]: Detected new or changed devices on smithi148 2024-09-21T12:22:40.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:40 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:40.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:40 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:22:40.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:40 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:22:40.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:40 smithi078 ceph-mon[23681]: Adjusting osd_memory_target on smithi148 to 7968M 2024-09-21T12:22:40.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:40 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:40.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:40 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:40.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:40 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:22:40.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:40 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:22:40.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:40 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:22:41.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:41 smithi148 ceph-mon[25818]: pgmap v134: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-21T12:22:41.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:41 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:41.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:41 smithi148 ceph-mon[25818]: from='client.24169 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi148:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:22:41.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:41 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T12:22:41.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:41 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T12:22:41.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:41 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:22:41.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:41 smithi078 ceph-mon[23681]: pgmap v134: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-21T12:22:41.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:41 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:41.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:41 smithi078 ceph-mon[23681]: from='client.24169 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi148:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:22:41.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:41 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T12:22:41.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:41 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T12:22:41.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:41 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:22:43.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:43 smithi148 ceph-mon[25818]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 88 KiB/s, 0 objects/s recovering 2024-09-21T12:22:43.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:43 smithi148 ceph-mon[25818]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T12:22:43.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:43 smithi148 ceph-mon[25818]: Cluster is now healthy 2024-09-21T12:22:43.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:43 smithi148 ceph-mon[25818]: from='client.? 172.21.15.148:0/438594709' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "72f25889-26f5-4464-9ce9-90f163da2fe2"}]: dispatch 2024-09-21T12:22:43.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:43 smithi148 ceph-mon[25818]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "72f25889-26f5-4464-9ce9-90f163da2fe2"}]: dispatch 2024-09-21T12:22:43.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:43 smithi148 ceph-mon[25818]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "72f25889-26f5-4464-9ce9-90f163da2fe2"}]': finished 2024-09-21T12:22:43.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:43 smithi148 ceph-mon[25818]: osdmap e44: 7 total, 6 up, 7 in 2024-09-21T12:22:43.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:43 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T12:22:43.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:43 smithi078 ceph-mon[23681]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 88 KiB/s, 0 objects/s recovering 2024-09-21T12:22:43.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:43 smithi078 ceph-mon[23681]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T12:22:43.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:43 smithi078 ceph-mon[23681]: Cluster is now healthy 2024-09-21T12:22:43.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:43 smithi078 ceph-mon[23681]: from='client.? 172.21.15.148:0/438594709' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "72f25889-26f5-4464-9ce9-90f163da2fe2"}]: dispatch 2024-09-21T12:22:43.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:43 smithi078 ceph-mon[23681]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "72f25889-26f5-4464-9ce9-90f163da2fe2"}]: dispatch 2024-09-21T12:22:43.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:43 smithi078 ceph-mon[23681]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "72f25889-26f5-4464-9ce9-90f163da2fe2"}]': finished 2024-09-21T12:22:43.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:43 smithi078 ceph-mon[23681]: osdmap e44: 7 total, 6 up, 7 in 2024-09-21T12:22:43.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:43 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T12:22:44.424 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:44 smithi148 ceph-mon[25818]: from='client.? 172.21.15.148:0/137369760' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T12:22:44.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:44 smithi078 ceph-mon[23681]: from='client.? 172.21.15.148:0/137369760' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T12:22:45.353 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:45 smithi148 ceph-mon[25818]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-21T12:22:45.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:45 smithi078 ceph-mon[23681]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-21T12:22:47.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:47 smithi148 ceph-mon[25818]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-09-21T12:22:47.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:47 smithi078 ceph-mon[23681]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-09-21T12:22:49.379 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:49 smithi148 ceph-mon[25818]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T12:22:49.379 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:49 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-21T12:22:49.379 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:49 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:22:49.379 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:49 smithi148 ceph-mon[25818]: Deploying daemon osd.6 on smithi148 2024-09-21T12:22:49.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:49 smithi078 ceph-mon[23681]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T12:22:49.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:49 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-21T12:22:49.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:49 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:22:49.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:49 smithi078 ceph-mon[23681]: Deploying daemon osd.6 on smithi148 2024-09-21T12:22:51.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:51 smithi148 ceph-mon[25818]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-21T12:22:51.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:51 smithi078 ceph-mon[23681]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-21T12:22:53.205 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:53 smithi148 ceph-mon[25818]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T12:22:53.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:53 smithi078 ceph-mon[23681]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T12:22:55.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:54 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:55.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:54 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:22:55.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:54 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:22:55.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:54 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:22:55.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:54 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:55.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:54 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:22:55.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:54 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:22:55.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:54 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:22:56.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:55 smithi148 ceph-mon[25818]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T12:22:56.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:55 smithi078 ceph-mon[23681]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T12:22:57.483 INFO:teuthology.orchestra.run.smithi148.stdout:Created osd(s) 6 on host 'smithi148' 2024-09-21T12:22:58.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:57 smithi148 ceph-mon[25818]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T12:22:58.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:57 smithi148 ceph-mon[25818]: from='osd.6 [v2:172.21.15.148:6816/1251772787,v1:172.21.15.148:6817/1251772787]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T12:22:58.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:57 smithi148 ceph-mon[25818]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T12:22:58.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:57 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:58.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:57 smithi078 ceph-mon[23681]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T12:22:58.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:57 smithi078 ceph-mon[23681]: from='osd.6 [v2:172.21.15.148:6816/1251772787,v1:172.21.15.148:6817/1251772787]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T12:22:58.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:57 smithi078 ceph-mon[23681]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T12:22:58.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:57 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:22:58.172 DEBUG:teuthology.orchestra.run.smithi148:osd.6> sudo journalctl -f -n 0 -u ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.6.service 2024-09-21T12:22:58.174 INFO:tasks.cephadm:Deploying osd.7 on smithi148 with /dev/vg_nvme/lv_1... 2024-09-21T12:22:58.174 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_1 2024-09-21T12:22:58.903 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:58 smithi148 ceph-mon[25818]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T12:22:58.903 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:58 smithi148 ceph-mon[25818]: osdmap e45: 7 total, 6 up, 7 in 2024-09-21T12:22:58.903 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:58 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T12:22:58.903 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:58 smithi148 ceph-mon[25818]: from='osd.6 [v2:172.21.15.148:6816/1251772787,v1:172.21.15.148:6817/1251772787]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-09-21T12:22:58.903 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:58 smithi148 ceph-mon[25818]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-09-21T12:22:58.903 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:22:58 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[35990]: 2024-09-21T12:22:58.800+0000 7f0f0f6cb700 -1 osd.6 0 waiting for initial osdmap 2024-09-21T12:22:58.903 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:22:58 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[35990]: 2024-09-21T12:22:58.806+0000 7f0f0805e700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T12:22:59.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:58 smithi078 ceph-mon[23681]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T12:22:59.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:58 smithi078 ceph-mon[23681]: osdmap e45: 7 total, 6 up, 7 in 2024-09-21T12:22:59.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:58 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T12:22:59.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:58 smithi078 ceph-mon[23681]: from='osd.6 [v2:172.21.15.148:6816/1251772787,v1:172.21.15.148:6817/1251772787]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-09-21T12:22:59.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:58 smithi078 ceph-mon[23681]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-09-21T12:22:59.898 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:59 smithi148 ceph-mon[25818]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T12:22:59.898 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:59 smithi148 ceph-mon[25818]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]': finished 2024-09-21T12:22:59.899 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:59 smithi148 ceph-mon[25818]: osdmap e46: 7 total, 6 up, 7 in 2024-09-21T12:22:59.899 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:59 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T12:22:59.899 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:22:59 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T12:23:00.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:59 smithi078 ceph-mon[23681]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-21T12:23:00.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:59 smithi078 ceph-mon[23681]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]': finished 2024-09-21T12:23:00.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:59 smithi078 ceph-mon[23681]: osdmap e46: 7 total, 6 up, 7 in 2024-09-21T12:23:00.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:59 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T12:23:00.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:22:59 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T12:23:01.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:00 smithi148 ceph-mon[25818]: purged_snaps scrub starts 2024-09-21T12:23:01.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:00 smithi148 ceph-mon[25818]: purged_snaps scrub ok 2024-09-21T12:23:01.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:00 smithi148 ceph-mon[25818]: osd.6 [v2:172.21.15.148:6816/1251772787,v1:172.21.15.148:6817/1251772787] boot 2024-09-21T12:23:01.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:00 smithi148 ceph-mon[25818]: osdmap e47: 7 total, 7 up, 7 in 2024-09-21T12:23:01.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:00 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T12:23:01.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:00 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:23:01.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:00 smithi078 ceph-mon[23681]: purged_snaps scrub starts 2024-09-21T12:23:01.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:00 smithi078 ceph-mon[23681]: purged_snaps scrub ok 2024-09-21T12:23:01.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:00 smithi078 ceph-mon[23681]: osd.6 [v2:172.21.15.148:6816/1251772787,v1:172.21.15.148:6817/1251772787] boot 2024-09-21T12:23:01.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:00 smithi078 ceph-mon[23681]: osdmap e47: 7 total, 7 up, 7 in 2024-09-21T12:23:01.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:00 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T12:23:01.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:00 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:23:02.013 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:01 smithi148 ceph-mon[25818]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-21T12:23:02.014 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:01 smithi148 ceph-mon[25818]: osdmap e48: 7 total, 7 up, 7 in 2024-09-21T12:23:02.017 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-09-21T12:23:02.042 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi148:vg_nvme/lv_1 2024-09-21T12:23:02.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:01 smithi078 ceph-mon[23681]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-21T12:23:02.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:01 smithi078 ceph-mon[23681]: osdmap e48: 7 total, 7 up, 7 in 2024-09-21T12:23:03.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:02 smithi148 ceph-mon[25818]: osdmap e49: 7 total, 7 up, 7 in 2024-09-21T12:23:03.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:02 smithi148 ceph-mon[25818]: osdmap e50: 7 total, 7 up, 7 in 2024-09-21T12:23:03.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:02 smithi078 ceph-mon[23681]: osdmap e49: 7 total, 7 up, 7 in 2024-09-21T12:23:03.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:02 smithi078 ceph-mon[23681]: osdmap e50: 7 total, 7 up, 7 in 2024-09-21T12:23:03.847 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:03 smithi148 ceph-mon[25818]: pgmap v151: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-21T12:23:03.847 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:03 smithi148 ceph-mon[25818]: osdmap e51: 7 total, 7 up, 7 in 2024-09-21T12:23:04.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:03 smithi078 ceph-mon[23681]: pgmap v151: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-21T12:23:04.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:03 smithi078 ceph-mon[23681]: osdmap e51: 7 total, 7 up, 7 in 2024-09-21T12:23:05.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:05 smithi148 ceph-mon[25818]: pgmap v154: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-21T12:23:05.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:05 smithi148 ceph-mon[25818]: Detected new or changed devices on smithi148 2024-09-21T12:23:05.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:05 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:23:05.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:05 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:23:05.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:05 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:23:05.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:05 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:23:05.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:05 smithi148 ceph-mon[25818]: Adjusting osd_memory_target on smithi148 to 5312M 2024-09-21T12:23:05.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:05 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:23:05.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:05 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:23:05.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:05 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:23:05.802 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:05 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:23:05.802 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:05 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:23:05.802 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:05 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:23:05.802 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:05 smithi148 ceph-mon[25818]: from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi148:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:23:05.802 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:05 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T12:23:05.802 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:05 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T12:23:05.802 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:05 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:23:05.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:05 smithi078 ceph-mon[23681]: pgmap v154: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-21T12:23:05.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:05 smithi078 ceph-mon[23681]: Detected new or changed devices on smithi148 2024-09-21T12:23:05.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:05 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:23:05.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:05 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:23:05.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:05 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:23:05.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:05 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:23:05.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:05 smithi078 ceph-mon[23681]: Adjusting osd_memory_target on smithi148 to 5312M 2024-09-21T12:23:05.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:05 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:23:05.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:05 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:23:05.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:05 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:23:05.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:05 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:23:05.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:05 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:23:05.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:05 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:23:05.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:05 smithi078 ceph-mon[23681]: from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi148:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:23:05.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:05 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T12:23:05.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:05 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T12:23:05.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:05 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:23:07.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:07 smithi148 ceph-mon[25818]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 99 KiB/s, 0 objects/s recovering 2024-09-21T12:23:07.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:07 smithi148 ceph-mon[25818]: from='client.? 172.21.15.148:0/1459174513' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bb8f1c3c-092c-4dec-b6fc-f94f7e34ca89"}]: dispatch 2024-09-21T12:23:07.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:07 smithi148 ceph-mon[25818]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bb8f1c3c-092c-4dec-b6fc-f94f7e34ca89"}]: dispatch 2024-09-21T12:23:07.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:07 smithi148 ceph-mon[25818]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bb8f1c3c-092c-4dec-b6fc-f94f7e34ca89"}]': finished 2024-09-21T12:23:07.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:07 smithi148 ceph-mon[25818]: osdmap e52: 8 total, 7 up, 8 in 2024-09-21T12:23:07.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:07 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T12:23:07.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:07 smithi078 ceph-mon[23681]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 99 KiB/s, 0 objects/s recovering 2024-09-21T12:23:07.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:07 smithi078 ceph-mon[23681]: from='client.? 172.21.15.148:0/1459174513' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bb8f1c3c-092c-4dec-b6fc-f94f7e34ca89"}]: dispatch 2024-09-21T12:23:07.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:07 smithi078 ceph-mon[23681]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bb8f1c3c-092c-4dec-b6fc-f94f7e34ca89"}]: dispatch 2024-09-21T12:23:07.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:07 smithi078 ceph-mon[23681]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bb8f1c3c-092c-4dec-b6fc-f94f7e34ca89"}]': finished 2024-09-21T12:23:07.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:07 smithi078 ceph-mon[23681]: osdmap e52: 8 total, 7 up, 8 in 2024-09-21T12:23:07.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:07 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T12:23:08.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:08 smithi148 ceph-mon[25818]: from='client.? 172.21.15.148:0/2028997876' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T12:23:08.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:08 smithi078 ceph-mon[23681]: from='client.? 172.21.15.148:0/2028997876' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T12:23:09.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:09 smithi148 ceph-mon[25818]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-21T12:23:09.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:09 smithi078 ceph-mon[23681]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-21T12:23:11.772 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:11 smithi148 ceph-mon[25818]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-21T12:23:11.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:11 smithi078 ceph-mon[23681]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-21T12:23:13.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:13 smithi148 ceph-mon[25818]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T12:23:13.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:13 smithi078 ceph-mon[23681]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T12:23:14.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:14 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-21T12:23:14.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:14 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:23:14.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:14 smithi148 ceph-mon[25818]: Deploying daemon osd.7 on smithi148 2024-09-21T12:23:14.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:14 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/trash_purge_schedule"}]: dispatch 2024-09-21T12:23:14.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:14 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/mirror_snapshot_schedule"}]: dispatch 2024-09-21T12:23:14.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-21T12:23:14.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:23:14.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:14 smithi078 ceph-mon[23681]: Deploying daemon osd.7 on smithi148 2024-09-21T12:23:14.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/trash_purge_schedule"}]: dispatch 2024-09-21T12:23:14.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/mirror_snapshot_schedule"}]: dispatch 2024-09-21T12:23:15.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:15 smithi148 ceph-mon[25818]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-21T12:23:15.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:15 smithi078 ceph-mon[23681]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-21T12:23:17.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:17 smithi148 ceph-mon[25818]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T12:23:17.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:17 smithi078 ceph-mon[23681]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T12:23:20.235 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:19 smithi148 ceph-mon[25818]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T12:23:20.236 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:19 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:23:20.236 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:19 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:23:20.236 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:19 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:23:20.236 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:19 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:23:20.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:19 smithi078 ceph-mon[23681]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T12:23:20.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:19 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:23:20.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:19 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:23:20.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:19 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:23:20.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:19 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:23:22.021 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:21 smithi148 ceph-mon[25818]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T12:23:22.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:21 smithi078 ceph-mon[23681]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T12:23:22.554 INFO:teuthology.orchestra.run.smithi148.stdout:Created osd(s) 7 on host 'smithi148' 2024-09-21T12:23:23.201 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:22 smithi148 ceph-mon[25818]: from='osd.7 [v2:172.21.15.148:6824/1735375122,v1:172.21.15.148:6825/1735375122]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T12:23:23.201 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:22 smithi148 ceph-mon[25818]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T12:23:23.201 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:22 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:23:23.226 DEBUG:teuthology.orchestra.run.smithi148:osd.7> sudo journalctl -f -n 0 -u ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.7.service 2024-09-21T12:23:23.229 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-21T12:23:23.229 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph osd stat -f json 2024-09-21T12:23:23.256 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:22 smithi078 ceph-mon[23681]: from='osd.7 [v2:172.21.15.148:6824/1735375122,v1:172.21.15.148:6825/1735375122]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T12:23:23.256 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:22 smithi078 ceph-mon[23681]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T12:23:23.257 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:22 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:23:24.167 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:23 smithi148 ceph-mon[25818]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T12:23:24.168 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:23 smithi148 ceph-mon[25818]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T12:23:24.168 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:23 smithi148 ceph-mon[25818]: osdmap e53: 8 total, 7 up, 8 in 2024-09-21T12:23:24.168 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:23 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T12:23:24.168 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:23 smithi148 ceph-mon[25818]: from='osd.7 [v2:172.21.15.148:6824/1735375122,v1:172.21.15.148:6825/1735375122]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-09-21T12:23:24.168 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:23 smithi148 ceph-mon[25818]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-09-21T12:23:24.168 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:23 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:23:24.169 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:23:23 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[38978]: 2024-09-21T12:23:23.917+0000 7ff7df041700 -1 osd.7 0 waiting for initial osdmap 2024-09-21T12:23:24.169 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:23:23 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[38978]: 2024-09-21T12:23:23.925+0000 7ff7d99d8700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T12:23:24.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:23 smithi078 ceph-mon[23681]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T12:23:24.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:23 smithi078 ceph-mon[23681]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T12:23:24.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:23 smithi078 ceph-mon[23681]: osdmap e53: 8 total, 7 up, 8 in 2024-09-21T12:23:24.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:23 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T12:23:24.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:23 smithi078 ceph-mon[23681]: from='osd.7 [v2:172.21.15.148:6824/1735375122,v1:172.21.15.148:6825/1735375122]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-09-21T12:23:24.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:23 smithi078 ceph-mon[23681]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-09-21T12:23:24.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:23 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:23:25.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:24 smithi078 ceph-mon[23681]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]': finished 2024-09-21T12:23:25.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:24 smithi078 ceph-mon[23681]: osdmap e54: 8 total, 7 up, 8 in 2024-09-21T12:23:25.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:24 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T12:23:25.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:24 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T12:23:25.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:24 smithi078 ceph-mon[23681]: osd.7 [v2:172.21.15.148:6824/1735375122,v1:172.21.15.148:6825/1735375122] boot 2024-09-21T12:23:25.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:24 smithi078 ceph-mon[23681]: osdmap e55: 8 total, 8 up, 8 in 2024-09-21T12:23:25.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:24 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T12:23:25.184 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:24 smithi148 ceph-mon[25818]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]': finished 2024-09-21T12:23:25.185 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:24 smithi148 ceph-mon[25818]: osdmap e54: 8 total, 7 up, 8 in 2024-09-21T12:23:25.185 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:24 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T12:23:25.185 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:24 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T12:23:25.185 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:24 smithi148 ceph-mon[25818]: osd.7 [v2:172.21.15.148:6824/1735375122,v1:172.21.15.148:6825/1735375122] boot 2024-09-21T12:23:25.185 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:24 smithi148 ceph-mon[25818]: osdmap e55: 8 total, 8 up, 8 in 2024-09-21T12:23:25.185 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:24 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T12:23:25.256 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T12:23:26.068 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1726921404,"num_in_osds":8,"osd_in_since":1726921387,"num_remapped_pgs":1} 2024-09-21T12:23:26.069 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T12:23:26.266 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:25 smithi078 ceph-mon[23681]: purged_snaps scrub starts 2024-09-21T12:23:26.267 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:25 smithi078 ceph-mon[23681]: purged_snaps scrub ok 2024-09-21T12:23:26.267 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:25 smithi078 ceph-mon[23681]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T12:23:26.267 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:25 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/3109391713' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-21T12:23:26.267 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:25 smithi078 ceph-mon[23681]: osdmap e56: 8 total, 8 up, 8 in 2024-09-21T12:23:26.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:25 smithi148 ceph-mon[25818]: purged_snaps scrub starts 2024-09-21T12:23:26.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:25 smithi148 ceph-mon[25818]: purged_snaps scrub ok 2024-09-21T12:23:26.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:25 smithi148 ceph-mon[25818]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-21T12:23:26.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:25 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/3109391713' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-21T12:23:26.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:25 smithi148 ceph-mon[25818]: osdmap e56: 8 total, 8 up, 8 in 2024-09-21T12:23:26.321 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/mon.smithi078/config 2024-09-21T12:23:28.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:27 smithi078 ceph-mon[23681]: pgmap v170: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:23:28.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:27 smithi078 ceph-mon[23681]: Detected new or changed devices on smithi148 2024-09-21T12:23:28.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:27 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:23:28.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:27 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:23:28.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:27 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:23:28.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:27 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:23:28.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:27 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:23:28.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:27 smithi078 ceph-mon[23681]: Adjusting osd_memory_target on smithi148 to 3984M 2024-09-21T12:23:28.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:27 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:23:28.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:27 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:23:28.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:27 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:23:28.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:27 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:23:28.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:27 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:23:28.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:27 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:23:28.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:27 smithi078 ceph-mon[23681]: osdmap e57: 8 total, 8 up, 8 in 2024-09-21T12:23:28.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:27 smithi078 ceph-mon[23681]: osdmap e58: 8 total, 8 up, 8 in 2024-09-21T12:23:28.176 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T12:23:28.176 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":59,"fsid":"6263deca-7813-11ef-baf6-efdc52797490","created":"2024-09-21T12:17:26.021437+0000","modified":"2024-09-21T12:23:28.103089+0000","last_up_change":"2024-09-21T12:23:24.910556+0000","last_in_change":"2024-09-21T12:23:07.132097+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T12:21:24.290828+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"908d7633-5010-4ace-8275-e8e2c504b0f7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":2381128776},{"type":"v1","addr":"172.21.15.78:6803","nonce":2381128776}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":2381128776},{"type":"v1","addr":"172.21.15.78:6805","nonce":2381128776}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":2381128776},{"type":"v1","addr":"172.21.15.78:6809","nonce":2381128776}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":2381128776},{"type":"v1","addr":"172.21.15.78:6807","nonce":2381128776}]},"public_addr":"172.21.15.78:6803/2381128776","cluster_addr":"172.21.15.78:6805/2381128776","heartbeat_back_addr":"172.21.15.78:6809/2381128776","heartbeat_front_addr":"172.21.15.78:6807/2381128776","state":["exists","up"]},{"osd":1,"uuid":"71d611f1-03c6-471c-baa2-602d6f2bc697","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":2942847307},{"type":"v1","addr":"172.21.15.78:6811","nonce":2942847307}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":2942847307},{"type":"v1","addr":"172.21.15.78:6813","nonce":2942847307}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":2942847307},{"type":"v1","addr":"172.21.15.78:6817","nonce":2942847307}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":2942847307},{"type":"v1","addr":"172.21.15.78:6815","nonce":2942847307}]},"public_addr":"172.21.15.78:6811/2942847307","cluster_addr":"172.21.15.78:6813/2942847307","heartbeat_back_addr":"172.21.15.78:6817/2942847307","heartbeat_front_addr":"172.21.15.78:6815/2942847307","state":["exists","up"]},{"osd":2,"uuid":"c200aece-da2c-4ccb-8dd7-f73c40b5b5e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":1377778238},{"type":"v1","addr":"172.21.15.78:6819","nonce":1377778238}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":1377778238},{"type":"v1","addr":"172.21.15.78:6821","nonce":1377778238}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6824","nonce":1377778238},{"type":"v1","addr":"172.21.15.78:6825","nonce":1377778238}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":1377778238},{"type":"v1","addr":"172.21.15.78:6823","nonce":1377778238}]},"public_addr":"172.21.15.78:6819/1377778238","cluster_addr":"172.21.15.78:6821/1377778238","heartbeat_back_addr":"172.21.15.78:6825/1377778238","heartbeat_front_addr":"172.21.15.78:6823/1377778238","state":["exists","up"]},{"osd":3,"uuid":"7db813e7-9c1a-4f95-b4b7-c871a878cf15","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6826","nonce":880260351},{"type":"v1","addr":"172.21.15.78:6827","nonce":880260351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6828","nonce":880260351},{"type":"v1","addr":"172.21.15.78:6829","nonce":880260351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6832","nonce":880260351},{"type":"v1","addr":"172.21.15.78:6833","nonce":880260351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6830","nonce":880260351},{"type":"v1","addr":"172.21.15.78:6831","nonce":880260351}]},"public_addr":"172.21.15.78:6827/880260351","cluster_addr":"172.21.15.78:6829/880260351","heartbeat_back_addr":"172.21.15.78:6833/880260351","heartbeat_front_addr":"172.21.15.78:6831/880260351","state":["exists","up"]},{"osd":4,"uuid":"2c7f0c21-a23f-4039-9a82-8cd307394520","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6800","nonce":3618711344},{"type":"v1","addr":"172.21.15.148:6801","nonce":3618711344}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6802","nonce":3618711344},{"type":"v1","addr":"172.21.15.148:6803","nonce":3618711344}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6806","nonce":3618711344},{"type":"v1","addr":"172.21.15.148:6807","nonce":3618711344}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6804","nonce":3618711344},{"type":"v1","addr":"172.21.15.148:6805","nonce":3618711344}]},"public_addr":"172.21.15.148:6801/3618711344","cluster_addr":"172.21.15.148:6803/3618711344","heartbeat_back_addr":"172.21.15.148:6807/3618711344","heartbeat_front_addr":"172.21.15.148:6805/3618711344","state":["exists","up"]},{"osd":5,"uuid":"8c2fb91b-c05c-4f2d-a272-d15a98be394d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6808","nonce":1612483000},{"type":"v1","addr":"172.21.15.148:6809","nonce":1612483000}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6810","nonce":1612483000},{"type":"v1","addr":"172.21.15.148:6811","nonce":1612483000}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6814","nonce":1612483000},{"type":"v1","addr":"172.21.15.148:6815","nonce":1612483000}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6812","nonce":1612483000},{"type":"v1","addr":"172.21.15.148:6813","nonce":1612483000}]},"public_addr":"172.21.15.148:6809/1612483000","cluster_addr":"172.21.15.148:6811/1612483000","heartbeat_back_addr":"172.21.15.148:6815/1612483000","heartbeat_front_addr":"172.21.15.148:6813/1612483000","state":["exists","up"]},{"osd":6,"uuid":"72f25889-26f5-4464-9ce9-90f163da2fe2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6816","nonce":1251772787},{"type":"v1","addr":"172.21.15.148:6817","nonce":1251772787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6818","nonce":1251772787},{"type":"v1","addr":"172.21.15.148:6819","nonce":1251772787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6822","nonce":1251772787},{"type":"v1","addr":"172.21.15.148:6823","nonce":1251772787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6820","nonce":1251772787},{"type":"v1","addr":"172.21.15.148:6821","nonce":1251772787}]},"public_addr":"172.21.15.148:6817/1251772787","cluster_addr":"172.21.15.148:6819/1251772787","heartbeat_back_addr":"172.21.15.148:6823/1251772787","heartbeat_front_addr":"172.21.15.148:6821/1251772787","state":["exists","up"]},{"osd":7,"uuid":"bb8f1c3c-092c-4dec-b6fc-f94f7e34ca89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6824","nonce":1735375122},{"type":"v1","addr":"172.21.15.148:6825","nonce":1735375122}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6826","nonce":1735375122},{"type":"v1","addr":"172.21.15.148:6827","nonce":1735375122}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6830","nonce":1735375122},{"type":"v1","addr":"172.21.15.148:6831","nonce":1735375122}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6828","nonce":1735375122},{"type":"v1","addr":"172.21.15.148:6829","nonce":1735375122}]},"public_addr":"172.21.15.148:6825/1735375122","cluster_addr":"172.21.15.148:6827/1735375122","heartbeat_back_addr":"172.21.15.148:6831/1735375122","heartbeat_front_addr":"172.21.15.148:6829/1735375122","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T12:20:22.464598+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T12:20:54.658132+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T12:21:21.767843+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T12:21:47.207365+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T12:22:08.470583+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T12:22:33.350778+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T12:22:57.951645+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T12:23:23.024923+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.78:0/714474677":"2024-09-22T12:19:13.922617+0000","172.21.15.78:0/3936658010":"2024-09-22T12:19:13.922617+0000","172.21.15.78:0/2286386760":"2024-09-22T12:19:13.922617+0000","172.21.15.78:0/1245515466":"2024-09-22T12:18:22.586182+0000","172.21.15.78:6800/174900693":"2024-09-22T12:19:13.922617+0000","172.21.15.78:0/396700856":"2024-09-22T12:18:22.586182+0000","172.21.15.78:0/2465188523":"2024-09-22T12:18:22.586182+0000","172.21.15.78:6801/3921138491":"2024-09-22T12:18:22.586182+0000","172.21.15.78:6800/3921138491":"2024-09-22T12:18:22.586182+0000","172.21.15.78:6800/2638763027":"2024-09-22T12:17:52.677556+0000","172.21.15.78:0/3749940933":"2024-09-22T12:17:52.677556+0000","172.21.15.78:6801/2638763027":"2024-09-22T12:17:52.677556+0000","172.21.15.78:6801/174900693":"2024-09-22T12:19:13.922617+0000","172.21.15.78:0/1846143256":"2024-09-22T12:17:52.677556+0000","172.21.15.78:0/1858355127":"2024-09-22T12:17:52.677556+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T12:23:28.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:27 smithi148 ceph-mon[25818]: pgmap v170: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:23:28.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:27 smithi148 ceph-mon[25818]: Detected new or changed devices on smithi148 2024-09-21T12:23:28.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:27 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:23:28.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:27 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:23:28.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:27 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:23:28.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:27 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:23:28.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:27 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:23:28.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:27 smithi148 ceph-mon[25818]: Adjusting osd_memory_target on smithi148 to 3984M 2024-09-21T12:23:28.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:27 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:23:28.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:27 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:23:28.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:27 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:23:28.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:27 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:23:28.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:27 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:23:28.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:27 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:23:28.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:27 smithi148 ceph-mon[25818]: osdmap e57: 8 total, 8 up, 8 in 2024-09-21T12:23:28.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:27 smithi148 ceph-mon[25818]: osdmap e58: 8 total, 8 up, 8 in 2024-09-21T12:23:28.957 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-21T12:21:24.290828+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-21T12:23:28.957 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph osd pool get .mgr pg_num 2024-09-21T12:23:29.168 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/mon.smithi078/config 2024-09-21T12:23:29.288 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:29 smithi078 ceph-mon[23681]: pgmap v173: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:23:29.289 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:29 smithi078 ceph-mon[23681]: osdmap e59: 8 total, 8 up, 8 in 2024-09-21T12:23:29.289 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:29 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/3840438242' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T12:23:29.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:29 smithi148 ceph-mon[25818]: pgmap v173: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:23:29.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:29 smithi148 ceph-mon[25818]: osdmap e59: 8 total, 8 up, 8 in 2024-09-21T12:23:29.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:29 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/3840438242' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T12:23:31.085 INFO:teuthology.orchestra.run.smithi078.stdout:pg_num: 1 2024-09-21T12:23:31.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:31 smithi078 ceph-mon[23681]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 126 KiB/s, 0 objects/s recovering 2024-09-21T12:23:31.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:31 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/1842905588' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-21T12:23:31.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:31 smithi148 ceph-mon[25818]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 126 KiB/s, 0 objects/s recovering 2024-09-21T12:23:31.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:31 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/1842905588' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-21T12:23:31.837 INFO:tasks.cephadm:Setting up client nodes... 2024-09-21T12:23:31.837 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-21T12:23:33.367 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:33 smithi078 ceph-mon[23681]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-21T12:23:33.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:33 smithi148 ceph-mon[25818]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-21T12:23:33.842 INFO:teuthology.orchestra.run.smithi078.stdout:[client.0] 2024-09-21T12:23:33.842 INFO:teuthology.orchestra.run.smithi078.stdout: key = AQDFuu5mjtbyMRAAYYQbmnGY2fwxVdSC1KDh2g== 2024-09-21T12:23:34.365 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:34 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/1172527566' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T12:23:34.365 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:34 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/1172527566' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T12:23:34.519 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:34 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/1172527566' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T12:23:34.519 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:34 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/1172527566' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T12:23:34.545 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-09-21T12:23:34.546 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-21T12:23:34.546 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-21T12:23:34.584 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph config log 1 --format=json 2024-09-21T12:23:34.831 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/mon.smithi078/config 2024-09-21T12:23:36.026 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:35 smithi078 ceph-mon[23681]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-09-21T12:23:36.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:35 smithi148 ceph-mon[25818]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-09-21T12:23:36.692 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T12:23:37.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:36 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/4020178055' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-21T12:23:37.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:36 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/4020178055' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-21T12:23:37.433 INFO:teuthology.orchestra.run.smithi078.stdout:[{"version":21,"timestamp":"2024-09-21T12:23:26.816245+0000","name":"","changes":[{"name":"osd/host:smithi148/osd_memory_target","previous_value":"5570077900","new_value":"4177558425"}]}] 2024-09-21T12:23:37.433 INFO:tasks.ceph_manager:config epoch is 21 2024-09-21T12:23:37.433 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-21T12:23:37.433 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-21T12:23:37.434 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph mgr dump --format=json 2024-09-21T12:23:37.638 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/mon.smithi078/config 2024-09-21T12:23:38.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:37 smithi078 ceph-mon[23681]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T12:23:38.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:37 smithi148 ceph-mon[25818]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T12:23:39.529 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T12:23:40.247 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:39 smithi078 ceph-mon[23681]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-21T12:23:40.247 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:39 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/2214720446' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T12:23:40.274 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":19,"active_gid":14202,"active_name":"smithi078.ntwyan","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6800","nonce":169835844},{"type":"v1","addr":"172.21.15.78:6801","nonce":169835844}]},"active_addr":"172.21.15.78:6801/169835844","active_change":"2024-09-21T12:19:13.922900+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14222,"name":"smithi148.oyiqmx","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.78:8443/","prometheus":"http://172.21.15.78:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":3975851517}]},{"addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":2194712445}]},{"addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":3743955996}]},{"addrvec":[{"type":"v2","addr":"172.21.15.78:0","nonce":2481183140}]}]}} 2024-09-21T12:23:40.277 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-21T12:23:40.277 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-21T12:23:40.277 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T12:23:40.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:39 smithi148 ceph-mon[25818]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-21T12:23:40.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:39 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/2214720446' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T12:23:40.484 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/mon.smithi078/config 2024-09-21T12:23:42.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:41 smithi078 ceph-mon[23681]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-09-21T12:23:42.227 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T12:23:42.227 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":59,"fsid":"6263deca-7813-11ef-baf6-efdc52797490","created":"2024-09-21T12:17:26.021437+0000","modified":"2024-09-21T12:23:28.103089+0000","last_up_change":"2024-09-21T12:23:24.910556+0000","last_in_change":"2024-09-21T12:23:07.132097+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T12:21:24.290828+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"908d7633-5010-4ace-8275-e8e2c504b0f7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":2381128776},{"type":"v1","addr":"172.21.15.78:6803","nonce":2381128776}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":2381128776},{"type":"v1","addr":"172.21.15.78:6805","nonce":2381128776}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":2381128776},{"type":"v1","addr":"172.21.15.78:6809","nonce":2381128776}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":2381128776},{"type":"v1","addr":"172.21.15.78:6807","nonce":2381128776}]},"public_addr":"172.21.15.78:6803/2381128776","cluster_addr":"172.21.15.78:6805/2381128776","heartbeat_back_addr":"172.21.15.78:6809/2381128776","heartbeat_front_addr":"172.21.15.78:6807/2381128776","state":["exists","up"]},{"osd":1,"uuid":"71d611f1-03c6-471c-baa2-602d6f2bc697","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":2942847307},{"type":"v1","addr":"172.21.15.78:6811","nonce":2942847307}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":2942847307},{"type":"v1","addr":"172.21.15.78:6813","nonce":2942847307}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":2942847307},{"type":"v1","addr":"172.21.15.78:6817","nonce":2942847307}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":2942847307},{"type":"v1","addr":"172.21.15.78:6815","nonce":2942847307}]},"public_addr":"172.21.15.78:6811/2942847307","cluster_addr":"172.21.15.78:6813/2942847307","heartbeat_back_addr":"172.21.15.78:6817/2942847307","heartbeat_front_addr":"172.21.15.78:6815/2942847307","state":["exists","up"]},{"osd":2,"uuid":"c200aece-da2c-4ccb-8dd7-f73c40b5b5e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":1377778238},{"type":"v1","addr":"172.21.15.78:6819","nonce":1377778238}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":1377778238},{"type":"v1","addr":"172.21.15.78:6821","nonce":1377778238}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6824","nonce":1377778238},{"type":"v1","addr":"172.21.15.78:6825","nonce":1377778238}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":1377778238},{"type":"v1","addr":"172.21.15.78:6823","nonce":1377778238}]},"public_addr":"172.21.15.78:6819/1377778238","cluster_addr":"172.21.15.78:6821/1377778238","heartbeat_back_addr":"172.21.15.78:6825/1377778238","heartbeat_front_addr":"172.21.15.78:6823/1377778238","state":["exists","up"]},{"osd":3,"uuid":"7db813e7-9c1a-4f95-b4b7-c871a878cf15","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6826","nonce":880260351},{"type":"v1","addr":"172.21.15.78:6827","nonce":880260351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6828","nonce":880260351},{"type":"v1","addr":"172.21.15.78:6829","nonce":880260351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6832","nonce":880260351},{"type":"v1","addr":"172.21.15.78:6833","nonce":880260351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6830","nonce":880260351},{"type":"v1","addr":"172.21.15.78:6831","nonce":880260351}]},"public_addr":"172.21.15.78:6827/880260351","cluster_addr":"172.21.15.78:6829/880260351","heartbeat_back_addr":"172.21.15.78:6833/880260351","heartbeat_front_addr":"172.21.15.78:6831/880260351","state":["exists","up"]},{"osd":4,"uuid":"2c7f0c21-a23f-4039-9a82-8cd307394520","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6800","nonce":3618711344},{"type":"v1","addr":"172.21.15.148:6801","nonce":3618711344}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6802","nonce":3618711344},{"type":"v1","addr":"172.21.15.148:6803","nonce":3618711344}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6806","nonce":3618711344},{"type":"v1","addr":"172.21.15.148:6807","nonce":3618711344}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6804","nonce":3618711344},{"type":"v1","addr":"172.21.15.148:6805","nonce":3618711344}]},"public_addr":"172.21.15.148:6801/3618711344","cluster_addr":"172.21.15.148:6803/3618711344","heartbeat_back_addr":"172.21.15.148:6807/3618711344","heartbeat_front_addr":"172.21.15.148:6805/3618711344","state":["exists","up"]},{"osd":5,"uuid":"8c2fb91b-c05c-4f2d-a272-d15a98be394d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6808","nonce":1612483000},{"type":"v1","addr":"172.21.15.148:6809","nonce":1612483000}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6810","nonce":1612483000},{"type":"v1","addr":"172.21.15.148:6811","nonce":1612483000}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6814","nonce":1612483000},{"type":"v1","addr":"172.21.15.148:6815","nonce":1612483000}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6812","nonce":1612483000},{"type":"v1","addr":"172.21.15.148:6813","nonce":1612483000}]},"public_addr":"172.21.15.148:6809/1612483000","cluster_addr":"172.21.15.148:6811/1612483000","heartbeat_back_addr":"172.21.15.148:6815/1612483000","heartbeat_front_addr":"172.21.15.148:6813/1612483000","state":["exists","up"]},{"osd":6,"uuid":"72f25889-26f5-4464-9ce9-90f163da2fe2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6816","nonce":1251772787},{"type":"v1","addr":"172.21.15.148:6817","nonce":1251772787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6818","nonce":1251772787},{"type":"v1","addr":"172.21.15.148:6819","nonce":1251772787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6822","nonce":1251772787},{"type":"v1","addr":"172.21.15.148:6823","nonce":1251772787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6820","nonce":1251772787},{"type":"v1","addr":"172.21.15.148:6821","nonce":1251772787}]},"public_addr":"172.21.15.148:6817/1251772787","cluster_addr":"172.21.15.148:6819/1251772787","heartbeat_back_addr":"172.21.15.148:6823/1251772787","heartbeat_front_addr":"172.21.15.148:6821/1251772787","state":["exists","up"]},{"osd":7,"uuid":"bb8f1c3c-092c-4dec-b6fc-f94f7e34ca89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6824","nonce":1735375122},{"type":"v1","addr":"172.21.15.148:6825","nonce":1735375122}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6826","nonce":1735375122},{"type":"v1","addr":"172.21.15.148:6827","nonce":1735375122}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6830","nonce":1735375122},{"type":"v1","addr":"172.21.15.148:6831","nonce":1735375122}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6828","nonce":1735375122},{"type":"v1","addr":"172.21.15.148:6829","nonce":1735375122}]},"public_addr":"172.21.15.148:6825/1735375122","cluster_addr":"172.21.15.148:6827/1735375122","heartbeat_back_addr":"172.21.15.148:6831/1735375122","heartbeat_front_addr":"172.21.15.148:6829/1735375122","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T12:20:22.464598+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T12:20:54.658132+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T12:21:21.767843+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T12:21:47.207365+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T12:22:08.470583+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T12:22:33.350778+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T12:22:57.951645+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T12:23:23.024923+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.78:0/714474677":"2024-09-22T12:19:13.922617+0000","172.21.15.78:0/3936658010":"2024-09-22T12:19:13.922617+0000","172.21.15.78:0/2286386760":"2024-09-22T12:19:13.922617+0000","172.21.15.78:0/1245515466":"2024-09-22T12:18:22.586182+0000","172.21.15.78:6800/174900693":"2024-09-22T12:19:13.922617+0000","172.21.15.78:0/396700856":"2024-09-22T12:18:22.586182+0000","172.21.15.78:0/2465188523":"2024-09-22T12:18:22.586182+0000","172.21.15.78:6801/3921138491":"2024-09-22T12:18:22.586182+0000","172.21.15.78:6800/3921138491":"2024-09-22T12:18:22.586182+0000","172.21.15.78:6800/2638763027":"2024-09-22T12:17:52.677556+0000","172.21.15.78:0/3749940933":"2024-09-22T12:17:52.677556+0000","172.21.15.78:6801/2638763027":"2024-09-22T12:17:52.677556+0000","172.21.15.78:6801/174900693":"2024-09-22T12:19:13.922617+0000","172.21.15.78:0/1846143256":"2024-09-22T12:17:52.677556+0000","172.21.15.78:0/1858355127":"2024-09-22T12:17:52.677556+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T12:23:42.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:41 smithi148 ceph-mon[25818]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-09-21T12:23:42.879 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-21T12:23:42.879 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T12:23:43.086 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/mon.smithi078/config 2024-09-21T12:23:43.437 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:43 smithi078 ceph-mon[23681]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:23:43.437 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:43 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/2092729009' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T12:23:43.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:43 smithi148 ceph-mon[25818]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:23:43.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:43 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/2092729009' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T12:23:44.917 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T12:23:44.917 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":59,"fsid":"6263deca-7813-11ef-baf6-efdc52797490","created":"2024-09-21T12:17:26.021437+0000","modified":"2024-09-21T12:23:28.103089+0000","last_up_change":"2024-09-21T12:23:24.910556+0000","last_in_change":"2024-09-21T12:23:07.132097+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T12:21:24.290828+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"908d7633-5010-4ace-8275-e8e2c504b0f7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":2381128776},{"type":"v1","addr":"172.21.15.78:6803","nonce":2381128776}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":2381128776},{"type":"v1","addr":"172.21.15.78:6805","nonce":2381128776}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":2381128776},{"type":"v1","addr":"172.21.15.78:6809","nonce":2381128776}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":2381128776},{"type":"v1","addr":"172.21.15.78:6807","nonce":2381128776}]},"public_addr":"172.21.15.78:6803/2381128776","cluster_addr":"172.21.15.78:6805/2381128776","heartbeat_back_addr":"172.21.15.78:6809/2381128776","heartbeat_front_addr":"172.21.15.78:6807/2381128776","state":["exists","up"]},{"osd":1,"uuid":"71d611f1-03c6-471c-baa2-602d6f2bc697","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":2942847307},{"type":"v1","addr":"172.21.15.78:6811","nonce":2942847307}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":2942847307},{"type":"v1","addr":"172.21.15.78:6813","nonce":2942847307}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":2942847307},{"type":"v1","addr":"172.21.15.78:6817","nonce":2942847307}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":2942847307},{"type":"v1","addr":"172.21.15.78:6815","nonce":2942847307}]},"public_addr":"172.21.15.78:6811/2942847307","cluster_addr":"172.21.15.78:6813/2942847307","heartbeat_back_addr":"172.21.15.78:6817/2942847307","heartbeat_front_addr":"172.21.15.78:6815/2942847307","state":["exists","up"]},{"osd":2,"uuid":"c200aece-da2c-4ccb-8dd7-f73c40b5b5e8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":1377778238},{"type":"v1","addr":"172.21.15.78:6819","nonce":1377778238}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":1377778238},{"type":"v1","addr":"172.21.15.78:6821","nonce":1377778238}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6824","nonce":1377778238},{"type":"v1","addr":"172.21.15.78:6825","nonce":1377778238}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":1377778238},{"type":"v1","addr":"172.21.15.78:6823","nonce":1377778238}]},"public_addr":"172.21.15.78:6819/1377778238","cluster_addr":"172.21.15.78:6821/1377778238","heartbeat_back_addr":"172.21.15.78:6825/1377778238","heartbeat_front_addr":"172.21.15.78:6823/1377778238","state":["exists","up"]},{"osd":3,"uuid":"7db813e7-9c1a-4f95-b4b7-c871a878cf15","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6826","nonce":880260351},{"type":"v1","addr":"172.21.15.78:6827","nonce":880260351}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6828","nonce":880260351},{"type":"v1","addr":"172.21.15.78:6829","nonce":880260351}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6832","nonce":880260351},{"type":"v1","addr":"172.21.15.78:6833","nonce":880260351}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6830","nonce":880260351},{"type":"v1","addr":"172.21.15.78:6831","nonce":880260351}]},"public_addr":"172.21.15.78:6827/880260351","cluster_addr":"172.21.15.78:6829/880260351","heartbeat_back_addr":"172.21.15.78:6833/880260351","heartbeat_front_addr":"172.21.15.78:6831/880260351","state":["exists","up"]},{"osd":4,"uuid":"2c7f0c21-a23f-4039-9a82-8cd307394520","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6800","nonce":3618711344},{"type":"v1","addr":"172.21.15.148:6801","nonce":3618711344}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6802","nonce":3618711344},{"type":"v1","addr":"172.21.15.148:6803","nonce":3618711344}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6806","nonce":3618711344},{"type":"v1","addr":"172.21.15.148:6807","nonce":3618711344}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6804","nonce":3618711344},{"type":"v1","addr":"172.21.15.148:6805","nonce":3618711344}]},"public_addr":"172.21.15.148:6801/3618711344","cluster_addr":"172.21.15.148:6803/3618711344","heartbeat_back_addr":"172.21.15.148:6807/3618711344","heartbeat_front_addr":"172.21.15.148:6805/3618711344","state":["exists","up"]},{"osd":5,"uuid":"8c2fb91b-c05c-4f2d-a272-d15a98be394d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6808","nonce":1612483000},{"type":"v1","addr":"172.21.15.148:6809","nonce":1612483000}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6810","nonce":1612483000},{"type":"v1","addr":"172.21.15.148:6811","nonce":1612483000}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6814","nonce":1612483000},{"type":"v1","addr":"172.21.15.148:6815","nonce":1612483000}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6812","nonce":1612483000},{"type":"v1","addr":"172.21.15.148:6813","nonce":1612483000}]},"public_addr":"172.21.15.148:6809/1612483000","cluster_addr":"172.21.15.148:6811/1612483000","heartbeat_back_addr":"172.21.15.148:6815/1612483000","heartbeat_front_addr":"172.21.15.148:6813/1612483000","state":["exists","up"]},{"osd":6,"uuid":"72f25889-26f5-4464-9ce9-90f163da2fe2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6816","nonce":1251772787},{"type":"v1","addr":"172.21.15.148:6817","nonce":1251772787}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6818","nonce":1251772787},{"type":"v1","addr":"172.21.15.148:6819","nonce":1251772787}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6822","nonce":1251772787},{"type":"v1","addr":"172.21.15.148:6823","nonce":1251772787}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6820","nonce":1251772787},{"type":"v1","addr":"172.21.15.148:6821","nonce":1251772787}]},"public_addr":"172.21.15.148:6817/1251772787","cluster_addr":"172.21.15.148:6819/1251772787","heartbeat_back_addr":"172.21.15.148:6823/1251772787","heartbeat_front_addr":"172.21.15.148:6821/1251772787","state":["exists","up"]},{"osd":7,"uuid":"bb8f1c3c-092c-4dec-b6fc-f94f7e34ca89","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6824","nonce":1735375122},{"type":"v1","addr":"172.21.15.148:6825","nonce":1735375122}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6826","nonce":1735375122},{"type":"v1","addr":"172.21.15.148:6827","nonce":1735375122}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6830","nonce":1735375122},{"type":"v1","addr":"172.21.15.148:6831","nonce":1735375122}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6828","nonce":1735375122},{"type":"v1","addr":"172.21.15.148:6829","nonce":1735375122}]},"public_addr":"172.21.15.148:6825/1735375122","cluster_addr":"172.21.15.148:6827/1735375122","heartbeat_back_addr":"172.21.15.148:6831/1735375122","heartbeat_front_addr":"172.21.15.148:6829/1735375122","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T12:20:22.464598+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T12:20:54.658132+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T12:21:21.767843+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T12:21:47.207365+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T12:22:08.470583+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T12:22:33.350778+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T12:22:57.951645+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T12:23:23.024923+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.78:0/714474677":"2024-09-22T12:19:13.922617+0000","172.21.15.78:0/3936658010":"2024-09-22T12:19:13.922617+0000","172.21.15.78:0/2286386760":"2024-09-22T12:19:13.922617+0000","172.21.15.78:0/1245515466":"2024-09-22T12:18:22.586182+0000","172.21.15.78:6800/174900693":"2024-09-22T12:19:13.922617+0000","172.21.15.78:0/396700856":"2024-09-22T12:18:22.586182+0000","172.21.15.78:0/2465188523":"2024-09-22T12:18:22.586182+0000","172.21.15.78:6801/3921138491":"2024-09-22T12:18:22.586182+0000","172.21.15.78:6800/3921138491":"2024-09-22T12:18:22.586182+0000","172.21.15.78:6800/2638763027":"2024-09-22T12:17:52.677556+0000","172.21.15.78:0/3749940933":"2024-09-22T12:17:52.677556+0000","172.21.15.78:6801/2638763027":"2024-09-22T12:17:52.677556+0000","172.21.15.78:6801/174900693":"2024-09-22T12:19:13.922617+0000","172.21.15.78:0/1846143256":"2024-09-22T12:17:52.677556+0000","172.21.15.78:0/1858355127":"2024-09-22T12:17:52.677556+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T12:23:45.227 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:45 smithi078 ceph-mon[23681]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:23:45.228 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:45 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/544856526' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T12:23:45.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:45 smithi148 ceph-mon[25818]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:23:45.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:45 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/544856526' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T12:23:45.662 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph tell osd.0 flush_pg_stats 2024-09-21T12:23:45.662 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph tell osd.1 flush_pg_stats 2024-09-21T12:23:45.663 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph tell osd.2 flush_pg_stats 2024-09-21T12:23:45.663 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph tell osd.3 flush_pg_stats 2024-09-21T12:23:45.664 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph tell osd.4 flush_pg_stats 2024-09-21T12:23:45.664 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph tell osd.5 flush_pg_stats 2024-09-21T12:23:45.664 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph tell osd.6 flush_pg_stats 2024-09-21T12:23:45.665 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph tell osd.7 flush_pg_stats 2024-09-21T12:23:46.080 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/mon.smithi078/config 2024-09-21T12:23:46.081 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/mon.smithi078/config 2024-09-21T12:23:46.084 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/mon.smithi078/config 2024-09-21T12:23:46.084 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/mon.smithi078/config 2024-09-21T12:23:46.086 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/mon.smithi078/config 2024-09-21T12:23:46.087 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/mon.smithi078/config 2024-09-21T12:23:46.088 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/mon.smithi078/config 2024-09-21T12:23:46.090 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/mon.smithi078/config 2024-09-21T12:23:47.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:47 smithi078 ceph-mon[23681]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:23:47.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:47 smithi148 ceph-mon[25818]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:23:49.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:49 smithi078 ceph-mon[23681]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:23:49.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:49 smithi148 ceph-mon[25818]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:23:51.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:51 smithi078 ceph-mon[23681]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:23:51.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:51 smithi148 ceph-mon[25818]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:23:53.163 INFO:teuthology.orchestra.run.smithi078.stdout:167503724562 2024-09-21T12:23:53.163 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.5 2024-09-21T12:23:53.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:53 smithi148 ceph-mon[25818]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:23:53.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:53 smithi078 ceph-mon[23681]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:23:54.728 INFO:teuthology.orchestra.run.smithi078.stdout:60129542182 2024-09-21T12:23:54.729 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.1 2024-09-21T12:23:55.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:55 smithi148 ceph-mon[25818]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:23:55.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:55 smithi078 ceph-mon[23681]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:23:57.533 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:57 smithi078 ceph-mon[23681]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:23:57.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:57 smithi148 ceph-mon[25818]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:23:57.556 INFO:teuthology.orchestra.run.smithi078.stdout:111669149725 2024-09-21T12:23:57.556 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.3 2024-09-21T12:23:59.057 INFO:teuthology.orchestra.run.smithi078.stdout:81604378658 2024-09-21T12:23:59.058 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.2 2024-09-21T12:23:59.437 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:23:59 smithi078 ceph-mon[23681]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:23:59.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:23:59 smithi148 ceph-mon[25818]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:01.086 INFO:teuthology.orchestra.run.smithi078.stdout:38654705710 2024-09-21T12:24:01.087 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.0 2024-09-21T12:24:01.226 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:01 smithi078 ceph-mon[23681]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:01.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:01 smithi148 ceph-mon[25818]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:01.602 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/mon.smithi078/config 2024-09-21T12:24:01.602 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/mon.smithi078/config 2024-09-21T12:24:01.603 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/mon.smithi078/config 2024-09-21T12:24:01.604 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/mon.smithi078/config 2024-09-21T12:24:01.617 INFO:teuthology.orchestra.run.smithi078.stdout:133143986200 2024-09-21T12:24:01.617 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.4 2024-09-21T12:24:03.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:03 smithi078 ceph-mon[23681]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:03.540 INFO:teuthology.orchestra.run.smithi078.stdout:236223201289 2024-09-21T12:24:03.541 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.7 2024-09-21T12:24:03.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:03 smithi148 ceph-mon[25818]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:03.826 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/mon.smithi078/config 2024-09-21T12:24:05.119 INFO:teuthology.orchestra.run.smithi078.stdout:201863462927 2024-09-21T12:24:05.119 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.6 2024-09-21T12:24:05.451 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:05 smithi078 ceph-mon[23681]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:05.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:05 smithi148 ceph-mon[25818]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:07.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:07 smithi078 ceph-mon[23681]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:07.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:07 smithi148 ceph-mon[25818]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:07.817 INFO:teuthology.orchestra.run.smithi078.stdout:111669149727 2024-09-21T12:24:08.458 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:08 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/507720022' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T12:24:08.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:08 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/507720022' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T12:24:08.875 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149725 got 111669149727 for osd.3 2024-09-21T12:24:08.875 DEBUG:teuthology.parallel:result is None 2024-09-21T12:24:09.504 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/mon.smithi078/config 2024-09-21T12:24:09.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:09 smithi148 ceph-mon[25818]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:09.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:09 smithi078 ceph-mon[23681]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:10.236 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/mon.smithi078/config 2024-09-21T12:24:10.416 INFO:teuthology.orchestra.run.smithi078.stdout:81604378661 2024-09-21T12:24:10.888 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/mon.smithi078/config 2024-09-21T12:24:11.059 INFO:teuthology.orchestra.run.smithi078.stdout:133143986202 2024-09-21T12:24:11.537 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:11 smithi078 ceph-mon[23681]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:11.538 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:11 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/332595942' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T12:24:11.538 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:11 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/3388392691' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T12:24:11.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:11 smithi148 ceph-mon[25818]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:11.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:11 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/332595942' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T12:24:11.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:11 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/3388392691' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T12:24:11.866 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986200 got 133143986202 for osd.4 2024-09-21T12:24:11.867 DEBUG:teuthology.parallel:result is None 2024-09-21T12:24:12.469 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378658 got 81604378661 for osd.2 2024-09-21T12:24:12.469 DEBUG:teuthology.parallel:result is None 2024-09-21T12:24:13.277 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:13 smithi078 ceph-mon[23681]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:13.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:13 smithi148 ceph-mon[25818]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:14.423 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:14 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/trash_purge_schedule"}]: dispatch 2024-09-21T12:24:14.423 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:14 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/mirror_snapshot_schedule"}]: dispatch 2024-09-21T12:24:14.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/trash_purge_schedule"}]: dispatch 2024-09-21T12:24:14.494 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/mirror_snapshot_schedule"}]: dispatch 2024-09-21T12:24:14.687 INFO:teuthology.orchestra.run.smithi078.stdout:60129542186 2024-09-21T12:24:15.283 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:15 smithi078 ceph-mon[23681]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:15.284 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:15 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/2523749261' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T12:24:15.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:15 smithi148 ceph-mon[25818]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:15.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:15 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/2523749261' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T12:24:15.678 INFO:teuthology.orchestra.run.smithi078.stdout:236223201291 2024-09-21T12:24:16.375 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201289 got 236223201291 for osd.7 2024-09-21T12:24:16.375 DEBUG:teuthology.parallel:result is None 2024-09-21T12:24:16.476 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:16 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/304832963' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T12:24:16.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:16 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/304832963' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T12:24:16.765 INFO:teuthology.orchestra.run.smithi078.stdout:167503724567 2024-09-21T12:24:16.913 INFO:teuthology.orchestra.run.smithi078.stdout:38654705714 2024-09-21T12:24:17.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:17 smithi148 ceph-mon[25818]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:17.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:17 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/659235785' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T12:24:17.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:17 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/2289783494' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T12:24:17.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:17 smithi078 ceph-mon[23681]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:17.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:17 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/659235785' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T12:24:17.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:17 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/2289783494' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T12:24:17.603 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542182 got 60129542186 for osd.1 2024-09-21T12:24:17.603 DEBUG:teuthology.parallel:result is None 2024-09-21T12:24:18.166 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724562 got 167503724567 for osd.5 2024-09-21T12:24:18.166 DEBUG:teuthology.parallel:result is None 2024-09-21T12:24:18.731 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705710 got 38654705714 for osd.0 2024-09-21T12:24:18.732 DEBUG:teuthology.parallel:result is None 2024-09-21T12:24:19.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:19 smithi078 ceph-mon[23681]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:19.438 INFO:teuthology.orchestra.run.smithi078.stdout:201863462930 2024-09-21T12:24:19.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:19 smithi148 ceph-mon[25818]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:20.077 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462927 got 201863462930 for osd.6 2024-09-21T12:24:20.077 DEBUG:teuthology.parallel:result is None 2024-09-21T12:24:20.077 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-21T12:24:20.077 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T12:24:20.291 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/mon.smithi078/config 2024-09-21T12:24:20.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:20 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/1091559807' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T12:24:20.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:20 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/1091559807' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T12:24:21.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:21 smithi078 ceph-mon[23681]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:21.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:21 smithi148 ceph-mon[25818]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:22.067 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T12:24:22.068 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2024-09-21T12:24:22.743 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":201,"stamp":"2024-09-21T12:24:22.025572+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48788,"kb_used_data":6292,"kb_used_omap":0,"kb_used_meta":42432,"kb_avail":749912428,"statfs":{"total":767960285184,"available":767910326272,"internally_reserved":0,"allocated":6443008,"data_stored":3546089,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43450368},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002164"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T12:23:28.108385+0000","last_change":"2024-09-21T12:23:28.108385+0000","last_active":"2024-09-21T12:23:28.108385+0000","last_peered":"2024-09-21T12:23:28.108385+0000","last_clean":"2024-09-21T12:23:28.108385+0000","last_became_active":"2024-09-21T12:23:28.107999+0000","last_became_peered":"2024-09-21T12:23:28.107999+0000","last_unstale":"2024-09-21T12:23:28.108385+0000","last_undegraded":"2024-09-21T12:23:28.108385+0000","last_fullsized":"2024-09-21T12:23:28.108385+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T12:21:24.740644+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T12:21:24.740644+0000","last_clean_scrub_stamp":"2024-09-21T12:21:24.740644+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:30:01.106086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3194880,"data_stored":3173568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201293,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6048,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739104,"statfs":{"total":95995035648,"available":95988842496,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52600000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47599999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55000000000000004}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61699999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64100000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60699999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70399999999999996}]}]},{"osd":6,"up_from":47,"seq":201863462931,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6168,"kb_used_data":1104,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738984,"statfs":{"total":95995035648,"available":95988719616,"internally_reserved":0,"allocated":1130496,"data_stored":773308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 12:23:59 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.70599999999999996}]},{"osd":1,"last update":"Sat Sep 21 12:23:59 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.67200000000000004}]},{"osd":2,"last update":"Sat Sep 21 12:23:59 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":5.6820000000000004,"5min":5.6820000000000004,"15min":5.6820000000000004},"last":0.77800000000000002}]},{"osd":3,"last update":"Sat Sep 21 12:23:59 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.622},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.0229999999999999,"15min":1.0229999999999999},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":11.85,"5min":11.85,"15min":11.85},"last":0.73299999999999998}]},{"osd":4,"last update":"Sat Sep 21 12:23:59 2024","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.57399999999999995}]},{"osd":5,"last update":"Sat Sep 21 12:23:59 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.79900000000000004},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.82099999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.75700000000000001}]}]},{"osd":1,"up_from":14,"seq":60129542187,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6360,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738792,"statfs":{"total":95995035648,"available":95988523008,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 12:24:02 2024","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":0.32500000000000001,"15min":0.30499999999999999},"min":{"1min":0.249,"5min":0.188,"15min":0.188},"max":{"1min":0.47299999999999998,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.24099999999999999},{"interface":"front","average":{"1min":0.35199999999999998,"5min":0.33900000000000002,"15min":0.31900000000000001},"min":{"1min":0.27300000000000002,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.46000000000000002,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.34799999999999998}]},{"osd":2,"last update":"Sat Sep 21 12:23:27 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.376,"15min":0.375},"min":{"1min":0.27500000000000002,"5min":0.221,"15min":0.221},"max":{"1min":0.56499999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.372},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.39600000000000002,"15min":0.39500000000000002},"min":{"1min":0.28899999999999998,"5min":0.254,"15min":0.254},"max":{"1min":0.51700000000000002,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.28599999999999998}]},{"osd":3,"last update":"Sat Sep 21 12:23:54 2024","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.41399999999999998,"15min":0.41799999999999998},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.57099999999999995,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.439,"15min":0.44700000000000001},"min":{"1min":0.32200000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.502,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.33000000000000002}]},{"osd":4,"last update":"Sat Sep 21 12:24:12 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.78900000000000003,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.56799999999999995,"15min":0.56399999999999995},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.44700000000000001}]},{"osd":5,"last update":"Sat Sep 21 12:23:36 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.58499999999999996}]},{"osd":6,"last update":"Sat Sep 21 12:24:02 2024","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.499}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48099999999999998}]}]},{"osd":0,"up_from":9,"seq":38654705715,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 21 12:23:58 2024","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.33600000000000002,"15min":0.313},"min":{"1min":0.22900000000000001,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":0.58499999999999996,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.34499999999999997,"15min":0.32100000000000001},"min":{"1min":0.21299999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.36399999999999999}]},{"osd":2,"last update":"Sat Sep 21 12:23:26 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.36699999999999999,"15min":0.35499999999999998},"min":{"1min":0.24099999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.38900000000000001,"15min":0.38400000000000001},"min":{"1min":0.30499999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"last":0.41999999999999998}]},{"osd":3,"last update":"Sat Sep 21 12:23:53 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.437,"15min":0.439},"min":{"1min":0.30099999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.27900000000000003,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.34899999999999998}]},{"osd":4,"last update":"Sat Sep 21 12:24:13 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.54900000000000004,"15min":0.54800000000000004},"min":{"1min":0.39800000000000002,"5min":0.379,"15min":0.379},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.378,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.78100000000000003,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.61699999999999999}]},{"osd":5,"last update":"Sat Sep 21 12:23:36 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.86299999999999999},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.85599999999999998}]},{"osd":6,"last update":"Sat Sep 21 12:24:01 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.68500000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72499999999999998}]}]},{"osd":2,"up_from":19,"seq":81604378663,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6232,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738920,"statfs":{"total":95995035648,"available":95988654080,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 12:23:30 2024","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.39100000000000001,"15min":0.39400000000000002},"min":{"1min":0.27000000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.52700000000000002,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.41799999999999998,"15min":0.41599999999999998},"min":{"1min":0.316,"5min":0.253,"15min":0.253},"max":{"1min":0.629,"5min":0.629,"15min":0.629},"last":0.40500000000000003}]},{"osd":1,"last update":"Sat Sep 21 12:23:30 2024","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.40600000000000003,"15min":0.40500000000000003},"min":{"1min":0.28399999999999997,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.44,"5min":0.42699999999999999,"15min":0.42499999999999999},"min":{"1min":0.318,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.29799999999999999}]},{"osd":3,"last update":"Sat Sep 21 12:23:51 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.43099999999999999,"15min":0.42899999999999999},"min":{"1min":0.32900000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.439,"15min":0.441},"min":{"1min":0.34399999999999997,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.57199999999999995,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.34200000000000003}]},{"osd":4,"last update":"Sat Sep 21 12:24:15 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.57799999999999996,"15min":0.56799999999999995},"min":{"1min":0.499,"5min":0.374,"15min":0.374},"max":{"1min":0.747,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.58099999999999996,"15min":0.56699999999999995},"min":{"1min":0.47299999999999998,"5min":0.371,"15min":0.371},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.51600000000000001}]},{"osd":5,"last update":"Sat Sep 21 12:23:39 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.57899999999999996}]},{"osd":6,"last update":"Sat Sep 21 12:24:00 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.505,"5min":0.505,"15min":0.505},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.46500000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42999999999999999}]}]},{"osd":3,"up_from":26,"seq":111669149730,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 12:23:50 2024","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.38,"15min":0.38100000000000001},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.54600000000000004,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.40300000000000002,"15min":0.40100000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.251}]},{"osd":1,"last update":"Sat Sep 21 12:23:50 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.58499999999999996,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.27700000000000002},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.624,"5min":0.624,"15min":0.624},"last":0.29599999999999999}]},{"osd":2,"last update":"Sat Sep 21 12:23:50 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47499999999999998,"15min":0.47399999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.23799999999999999}]},{"osd":4,"last update":"Sat Sep 21 12:24:10 2024","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.54300000000000004,"15min":0.52700000000000002},"min":{"1min":0.39600000000000002,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.628,"5min":0.55300000000000005,"15min":0.54100000000000004},"min":{"1min":0.432,"5min":0.374,"15min":0.374},"max":{"1min":0.98199999999999998,"5min":0.98199999999999998,"15min":0.98199999999999998},"last":0.504}]},{"osd":5,"last update":"Sat Sep 21 12:23:36 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.073,"5min":1.073,"15min":1.073},"last":0.51200000000000001}]},{"osd":6,"last update":"Sat Sep 21 12:24:00 2024","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.0429999999999999,"5min":1.0429999999999999,"15min":1.0429999999999999},"last":0.435},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.39700000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44500000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986205,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5716,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739436,"statfs":{"total":95995035648,"available":95989182464,"internally_reserved":0,"allocated":602112,"data_stored":244645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 12:24:17 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.67300000000000004,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.55500000000000005,"15min":0.55000000000000004},"min":{"1min":0.437,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.749,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.58599999999999997}]},{"osd":1,"last update":"Sat Sep 21 12:24:17 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.51600000000000001,"15min":0.51200000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.54300000000000004,"15min":0.53900000000000003},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.751,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.60199999999999998}]},{"osd":2,"last update":"Sat Sep 21 12:24:17 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.53800000000000003,"15min":0.53400000000000003},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.76600000000000001,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.56200000000000006},{"interface":"front","average":{"1min":1.3089999999999999,"5min":0.68100000000000005,"15min":0.57599999999999996},"min":{"1min":0.41899999999999998,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":14.827,"5min":14.827,"15min":14.827},"last":0.55400000000000005}]},{"osd":3,"last update":"Sat Sep 21 12:24:17 2024","interfaces":[{"interface":"back","average":{"1min":1.3180000000000001,"5min":0.69899999999999995,"15min":0.59599999999999997},"min":{"1min":0.40899999999999997,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":14.859,"5min":14.859,"15min":14.859},"last":0.629},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.54000000000000004,"15min":0.53600000000000003},"min":{"1min":0.46200000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.53700000000000003}]},{"osd":5,"last update":"Sat Sep 21 12:23:37 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.38,"15min":0.38},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.42099999999999999}]},{"osd":6,"last update":"Sat Sep 21 12:23:59 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.434},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.41099999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45300000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724569,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5660,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739492,"statfs":{"total":95995035648,"available":95989239808,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 12:23:38 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.66400000000000003}]},{"osd":1,"last update":"Sat Sep 21 12:23:38 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.51200000000000001}]},{"osd":2,"last update":"Sat Sep 21 12:23:38 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.53700000000000003}]},{"osd":3,"last update":"Sat Sep 21 12:23:38 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.57299999999999995}]},{"osd":4,"last update":"Sat Sep 21 12:23:38 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.64000000000000001}]},{"osd":6,"last update":"Sat Sep 21 12:24:02 2024","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.626},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.61899999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65300000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T12:24:22.744 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T12:24:22.954 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/mon.smithi078/config 2024-09-21T12:24:23.425 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:23 smithi078 ceph-mon[23681]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:23.425 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:23 smithi078 ceph-mon[23681]: from='client.14454 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T12:24:23.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:23 smithi148 ceph-mon[25818]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:23.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:23 smithi148 ceph-mon[25818]: from='client.14454 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T12:24:24.773 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T12:24:24.773 INFO:teuthology.orchestra.run.smithi078.stderr:dumped all 2024-09-21T12:24:25.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:25 smithi078 ceph-mon[23681]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:25.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:25 smithi078 ceph-mon[23681]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T12:24:25.401 INFO:teuthology.orchestra.run.smithi078.stdout:{"pg_ready":true,"pg_map":{"version":202,"stamp":"2024-09-21T12:24:24.025889+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48788,"kb_used_data":6292,"kb_used_omap":0,"kb_used_meta":42432,"kb_avail":749912428,"statfs":{"total":767960285184,"available":767910326272,"internally_reserved":0,"allocated":6443008,"data_stored":3546089,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43450368},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002117"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T12:23:28.108385+0000","last_change":"2024-09-21T12:23:28.108385+0000","last_active":"2024-09-21T12:23:28.108385+0000","last_peered":"2024-09-21T12:23:28.108385+0000","last_clean":"2024-09-21T12:23:28.108385+0000","last_became_active":"2024-09-21T12:23:28.107999+0000","last_became_peered":"2024-09-21T12:23:28.107999+0000","last_unstale":"2024-09-21T12:23:28.108385+0000","last_undegraded":"2024-09-21T12:23:28.108385+0000","last_fullsized":"2024-09-21T12:23:28.108385+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T12:21:24.740644+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T12:21:24.740644+0000","last_clean_scrub_stamp":"2024-09-21T12:21:24.740644+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:30:01.106086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3194880,"data_stored":3173568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201293,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6048,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739104,"statfs":{"total":95995035648,"available":95988842496,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52600000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47599999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55000000000000004}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61699999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64100000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60699999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70399999999999996}]}]},{"osd":6,"up_from":47,"seq":201863462931,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6168,"kb_used_data":1104,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738984,"statfs":{"total":95995035648,"available":95988719616,"internally_reserved":0,"allocated":1130496,"data_stored":773308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 12:23:59 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.70599999999999996}]},{"osd":1,"last update":"Sat Sep 21 12:23:59 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.67200000000000004}]},{"osd":2,"last update":"Sat Sep 21 12:23:59 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":5.6820000000000004,"5min":5.6820000000000004,"15min":5.6820000000000004},"last":0.77800000000000002}]},{"osd":3,"last update":"Sat Sep 21 12:23:59 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.622},{"interface":"front","average":{"1min":1.0229999999999999,"5min":1.0229999999999999,"15min":1.0229999999999999},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":11.85,"5min":11.85,"15min":11.85},"last":0.73299999999999998}]},{"osd":4,"last update":"Sat Sep 21 12:23:59 2024","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.57399999999999995}]},{"osd":5,"last update":"Sat Sep 21 12:23:59 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.79900000000000004},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.82099999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.75700000000000001}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6360,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738792,"statfs":{"total":95995035648,"available":95988523008,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 12:24:02 2024","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":0.32500000000000001,"15min":0.30499999999999999},"min":{"1min":0.249,"5min":0.188,"15min":0.188},"max":{"1min":0.47299999999999998,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.35199999999999998,"5min":0.33900000000000002,"15min":0.31900000000000001},"min":{"1min":0.27300000000000002,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.46000000000000002,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.51100000000000001}]},{"osd":2,"last update":"Sat Sep 21 12:23:27 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.376,"15min":0.375},"min":{"1min":0.27500000000000002,"5min":0.221,"15min":0.221},"max":{"1min":0.56499999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.39600000000000002,"15min":0.39500000000000002},"min":{"1min":0.28899999999999998,"5min":0.254,"15min":0.254},"max":{"1min":0.51700000000000002,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.45400000000000001}]},{"osd":3,"last update":"Sat Sep 21 12:23:54 2024","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.41399999999999998,"15min":0.41799999999999998},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.57099999999999995,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.439,"15min":0.44700000000000001},"min":{"1min":0.32200000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.502,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.44400000000000001}]},{"osd":4,"last update":"Sat Sep 21 12:24:12 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.78900000000000003,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.56799999999999995,"15min":0.56399999999999995},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.58899999999999997}]},{"osd":5,"last update":"Sat Sep 21 12:23:36 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.68200000000000005}]},{"osd":6,"last update":"Sat Sep 21 12:24:02 2024","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.745}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69799999999999995}]}]},{"osd":0,"up_from":9,"seq":38654705715,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 21 12:23:58 2024","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.33600000000000002,"15min":0.313},"min":{"1min":0.22900000000000001,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":0.58499999999999996,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.34499999999999997,"15min":0.32100000000000001},"min":{"1min":0.21299999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.36399999999999999}]},{"osd":2,"last update":"Sat Sep 21 12:23:26 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.36699999999999999,"15min":0.35499999999999998},"min":{"1min":0.24099999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.38900000000000001,"15min":0.38400000000000001},"min":{"1min":0.30499999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"last":0.41999999999999998}]},{"osd":3,"last update":"Sat Sep 21 12:23:53 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.437,"15min":0.439},"min":{"1min":0.30099999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.27900000000000003,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.34899999999999998}]},{"osd":4,"last update":"Sat Sep 21 12:24:13 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.54900000000000004,"15min":0.54800000000000004},"min":{"1min":0.39800000000000002,"5min":0.379,"15min":0.379},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.378,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.78100000000000003,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.61699999999999999}]},{"osd":5,"last update":"Sat Sep 21 12:23:36 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.86299999999999999},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.85599999999999998}]},{"osd":6,"last update":"Sat Sep 21 12:24:01 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.68500000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72499999999999998}]}]},{"osd":2,"up_from":19,"seq":81604378663,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6232,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738920,"statfs":{"total":95995035648,"available":95988654080,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 12:23:30 2024","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.39100000000000001,"15min":0.39400000000000002},"min":{"1min":0.27000000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.52700000000000002,"5min":0.57499999999999996,"15min":0.57499999999999996},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.41799999999999998,"15min":0.41599999999999998},"min":{"1min":0.316,"5min":0.253,"15min":0.253},"max":{"1min":0.629,"5min":0.629,"15min":0.629},"last":0.40500000000000003}]},{"osd":1,"last update":"Sat Sep 21 12:23:30 2024","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.40600000000000003,"15min":0.40500000000000003},"min":{"1min":0.28399999999999997,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.44,"5min":0.42699999999999999,"15min":0.42499999999999999},"min":{"1min":0.318,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"last":0.29799999999999999}]},{"osd":3,"last update":"Sat Sep 21 12:23:51 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.43099999999999999,"15min":0.42899999999999999},"min":{"1min":0.32900000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.439,"15min":0.441},"min":{"1min":0.34399999999999997,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.57199999999999995,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.34200000000000003}]},{"osd":4,"last update":"Sat Sep 21 12:24:15 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.57799999999999996,"15min":0.56799999999999995},"min":{"1min":0.499,"5min":0.374,"15min":0.374},"max":{"1min":0.747,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.58099999999999996,"15min":0.56699999999999995},"min":{"1min":0.47299999999999998,"5min":0.371,"15min":0.371},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.51600000000000001}]},{"osd":5,"last update":"Sat Sep 21 12:23:39 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.57899999999999996}]},{"osd":6,"last update":"Sat Sep 21 12:24:00 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.505,"5min":0.505,"15min":0.505},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.46500000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42999999999999999}]}]},{"osd":3,"up_from":26,"seq":111669149730,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 12:23:50 2024","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.38,"15min":0.38100000000000001},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.54600000000000004,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.40300000000000002,"15min":0.40100000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.251}]},{"osd":1,"last update":"Sat Sep 21 12:23:50 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.58499999999999996,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.27700000000000002},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.624,"5min":0.624,"15min":0.624},"last":0.29599999999999999}]},{"osd":2,"last update":"Sat Sep 21 12:23:50 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47499999999999998,"15min":0.47399999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.23799999999999999}]},{"osd":4,"last update":"Sat Sep 21 12:24:10 2024","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.54300000000000004,"15min":0.52700000000000002},"min":{"1min":0.39600000000000002,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.628,"5min":0.55300000000000005,"15min":0.54100000000000004},"min":{"1min":0.432,"5min":0.374,"15min":0.374},"max":{"1min":0.98199999999999998,"5min":0.98199999999999998,"15min":0.98199999999999998},"last":0.504}]},{"osd":5,"last update":"Sat Sep 21 12:23:36 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.073,"5min":1.073,"15min":1.073},"last":0.51200000000000001}]},{"osd":6,"last update":"Sat Sep 21 12:24:00 2024","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.0429999999999999,"5min":1.0429999999999999,"15min":1.0429999999999999},"last":0.435},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.39700000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44500000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986205,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5716,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739436,"statfs":{"total":95995035648,"available":95989182464,"internally_reserved":0,"allocated":602112,"data_stored":244645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 12:24:17 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.67300000000000004,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.55500000000000005,"15min":0.55000000000000004},"min":{"1min":0.437,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.749,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.58599999999999997}]},{"osd":1,"last update":"Sat Sep 21 12:24:17 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.51600000000000001,"15min":0.51200000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.54300000000000004,"15min":0.53900000000000003},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.751,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.60199999999999998}]},{"osd":2,"last update":"Sat Sep 21 12:24:17 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.53800000000000003,"15min":0.53400000000000003},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.76600000000000001,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.56200000000000006},{"interface":"front","average":{"1min":1.3089999999999999,"5min":0.68100000000000005,"15min":0.57599999999999996},"min":{"1min":0.41899999999999998,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":14.827,"5min":14.827,"15min":14.827},"last":0.55400000000000005}]},{"osd":3,"last update":"Sat Sep 21 12:24:17 2024","interfaces":[{"interface":"back","average":{"1min":1.3180000000000001,"5min":0.69899999999999995,"15min":0.59599999999999997},"min":{"1min":0.40899999999999997,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":14.859,"5min":14.859,"15min":14.859},"last":0.629},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.54000000000000004,"15min":0.53600000000000003},"min":{"1min":0.46200000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.53700000000000003}]},{"osd":5,"last update":"Sat Sep 21 12:23:37 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.38,"15min":0.38},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.42099999999999999}]},{"osd":6,"last update":"Sat Sep 21 12:23:59 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.434},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.41099999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45300000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724569,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5660,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739492,"statfs":{"total":95995035648,"available":95989239808,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 12:23:38 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.66400000000000003}]},{"osd":1,"last update":"Sat Sep 21 12:23:38 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.51200000000000001}]},{"osd":2,"last update":"Sat Sep 21 12:23:38 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.53700000000000003}]},{"osd":3,"last update":"Sat Sep 21 12:23:38 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.57299999999999995}]},{"osd":4,"last update":"Sat Sep 21 12:23:38 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.64000000000000001}]},{"osd":6,"last update":"Sat Sep 21 12:24:02 2024","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.626},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.61899999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65300000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T12:24:25.402 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-21T12:24:25.403 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-21T12:24:25.403 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-21T12:24:25.403 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph health --format=json 2024-09-21T12:24:25.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:25 smithi148 ceph-mon[25818]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:25.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:25 smithi148 ceph-mon[25818]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T12:24:25.610 INFO:teuthology.orchestra.run.smithi078.stderr:Inferring config /var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/mon.smithi078/config 2024-09-21T12:24:27.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:27 smithi078 ceph-mon[23681]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:27.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:27 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:24:27.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:27 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:24:27.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:27 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:24:27.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:27 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:24:27.461 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T12:24:27.461 INFO:teuthology.orchestra.run.smithi078.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-21T12:24:27.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:27 smithi148 ceph-mon[25818]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:27.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:27 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:24:27.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:27 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:24:27.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:27 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:24:27.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:27 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:24:28.131 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-21T12:24:28.131 INFO:tasks.cephadm:Setup complete, yielding 2024-09-21T12:24:28.131 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-21T12:24:28.140 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-21T12:24:28.140 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- bash -c 'ceph orch status' 2024-09-21T12:24:28.426 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:28 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/3530494713' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T12:24:28.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:28 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/3530494713' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T12:24:29.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:29 smithi148 ceph-mon[25818]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:29.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:29 smithi078 ceph-mon[23681]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:30.154 INFO:teuthology.orchestra.run.smithi078.stdout:Backend: cephadm 2024-09-21T12:24:30.154 INFO:teuthology.orchestra.run.smithi078.stdout:Available: Yes 2024-09-21T12:24:30.154 INFO:teuthology.orchestra.run.smithi078.stdout:Paused: No 2024-09-21T12:24:30.713 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- bash -c 'ceph orch ps' 2024-09-21T12:24:31.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:31 smithi078 ceph-mon[23681]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:31.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:31 smithi078 ceph-mon[23681]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:24:31.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:31 smithi148 ceph-mon[25818]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:31.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:31 smithi148 ceph-mon[25818]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:24:32.765 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T12:24:32.765 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (4m) 2m ago 5m 17.2M - ba2b418f427c 913ca01a7223 2024-09-21T12:24:32.765 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (5m) 2m ago 5m 6400k - 17.2.0 e1d6a67b021e 4a209e5da5ce 2024-09-21T12:24:32.766 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi148 smithi148 running (5m) 68s ago 5m 6405k - 17.2.0 e1d6a67b021e 0fa5e612307f 2024-09-21T12:24:32.766 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (4m) 2m ago 5m 44.2M - 8.3.5 dad864ee21e9 b6121d08c85b 2024-09-21T12:24:32.766 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.ntwyan smithi078 *:9283 running (6m) 2m ago 6m 447M - 17.2.0 e1d6a67b021e caeba9628014 2024-09-21T12:24:32.766 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi148.oyiqmx smithi148 *:8443,9283 running (5m) 68s ago 5m 411M - 17.2.0 e1d6a67b021e 47b925ccff59 2024-09-21T12:24:32.766 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (7m) 2m ago 7m 41.2M 2048M 17.2.0 e1d6a67b021e 8261b0b2c7e0 2024-09-21T12:24:32.766 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi148 smithi148 running (5m) 68s ago 4m 39.8M 2048M 17.2.0 e1d6a67b021e fda961db230f 2024-09-21T12:24:32.766 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (5m) 2m ago 5m 16.5M - 1dbe0e931976 e7ca2ce65161 2024-09-21T12:24:32.766 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi148 smithi148 *:9100 running (4m) 68s ago 4m 16.9M - 1dbe0e931976 1474152580ab 2024-09-21T12:24:32.766 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (4m) 2m ago 4m 33.4M 3216M 17.2.0 e1d6a67b021e d4d5e9977070 2024-09-21T12:24:32.766 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (3m) 2m ago 3m 37.2M 3216M 17.2.0 e1d6a67b021e b9a4b952233b 2024-09-21T12:24:32.766 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (3m) 2m ago 3m 31.0M 3216M 17.2.0 e1d6a67b021e 829fb8c2d827 2024-09-21T12:24:32.766 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (2m) 2m ago 2m 11.9M 3216M 17.2.0 e1d6a67b021e ef942c8556d0 2024-09-21T12:24:32.766 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi148 running (2m) 68s ago 2m 34.3M 3984M 17.2.0 e1d6a67b021e 62eefdd572c0 2024-09-21T12:24:32.767 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi148 running (2m) 68s ago 2m 33.0M 3984M 17.2.0 e1d6a67b021e 7b44be78baca 2024-09-21T12:24:32.767 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi148 running (99s) 68s ago 99s 31.3M 3984M 17.2.0 e1d6a67b021e 3d23057910ea 2024-09-21T12:24:32.767 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi148 running (74s) 68s ago 73s 10.2M 3984M 17.2.0 e1d6a67b021e f9e4db9d35fa 2024-09-21T12:24:32.767 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (4m) 2m ago 4m 53.2M - 514e6a882f6e cdcd63cf2477 2024-09-21T12:24:33.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:33 smithi078 ceph-mon[23681]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:33.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:33 smithi078 ceph-mon[23681]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:24:33.425 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- bash -c 'ceph orch ls' 2024-09-21T12:24:33.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:33 smithi148 ceph-mon[25818]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:33.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:33 smithi148 ceph-mon[25818]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:24:35.420 INFO:teuthology.orchestra.run.smithi078.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-09-21T12:24:35.420 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-09-21T12:24:35.420 INFO:teuthology.orchestra.run.smithi078.stdout:crash 2/2 2m ago 6m * 2024-09-21T12:24:35.421 INFO:teuthology.orchestra.run.smithi078.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-09-21T12:24:35.421 INFO:teuthology.orchestra.run.smithi078.stdout:mgr 2/2 2m ago 6m count:2 2024-09-21T12:24:35.421 INFO:teuthology.orchestra.run.smithi078.stdout:mon 2/2 2m ago 5m smithi078:172.21.15.78=smithi078;smithi148:172.21.15.148=smithi148;count:2 2024-09-21T12:24:35.421 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-09-21T12:24:35.421 INFO:teuthology.orchestra.run.smithi078.stdout:osd 8 2m ago - 2024-09-21T12:24:35.421 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-09-21T12:24:35.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:35 smithi148 ceph-mon[25818]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:35.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:35 smithi078 ceph-mon[23681]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:35.987 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- bash -c 'ceph orch host ls' 2024-09-21T12:24:36.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:36 smithi148 ceph-mon[25818]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:24:36.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:36 smithi078 ceph-mon[23681]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:24:37.296 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:37 smithi078 ceph-mon[23681]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:37.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:37 smithi148 ceph-mon[25818]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:37.986 INFO:teuthology.orchestra.run.smithi078.stdout:HOST ADDR LABELS STATUS 2024-09-21T12:24:37.986 INFO:teuthology.orchestra.run.smithi078.stdout:smithi078 172.21.15.78 2024-09-21T12:24:37.986 INFO:teuthology.orchestra.run.smithi078.stdout:smithi148 172.21.15.148 2024-09-21T12:24:37.986 INFO:teuthology.orchestra.run.smithi078.stdout:2 hosts in cluster 2024-09-21T12:24:38.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:38 smithi148 ceph-mon[25818]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:24:38.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:38 smithi078 ceph-mon[23681]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:24:38.594 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- bash -c 'ceph orch device ls' 2024-09-21T12:24:39.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:39 smithi148 ceph-mon[25818]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:39.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:39 smithi078 ceph-mon[23681]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:40.636 INFO:teuthology.orchestra.run.smithi078.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-09-21T12:24:40.637 INFO:teuthology.orchestra.run.smithi078.stdout:smithi078 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620400QR400BGN 400G LVM detected, locked 2024-09-21T12:24:40.637 INFO:teuthology.orchestra.run.smithi078.stdout:smithi148 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT6233005G400BGN 400G LVM detected, locked 2024-09-21T12:24:41.401 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-21T12:24:41.415 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-21T12:24:41.415 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-21T12:24:41.443 INFO:teuthology.orchestra.run.smithi078.stderr:+ systemctl stop nfs-server 2024-09-21T12:24:41.451 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi148.front.sepia.ceph.com 2024-09-21T12:24:41.451 DEBUG:teuthology.orchestra.run.smithi148:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-21T12:24:41.478 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:41 smithi148 ceph-mon[25818]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:41.478 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:41 smithi148 ceph-mon[25818]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:24:41.481 INFO:teuthology.orchestra.run.smithi148.stderr:+ systemctl stop nfs-server 2024-09-21T12:24:41.490 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-21T12:24:41.499 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-21T12:24:41.500 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- bash -c 'ceph fs volume create foofs' 2024-09-21T12:24:41.523 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:41 smithi078 ceph-mon[23681]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:41.523 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:41 smithi078 ceph-mon[23681]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:24:43.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:43 smithi078 ceph-mon[23681]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:43.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:43 smithi148 ceph-mon[25818]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:44.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:44 smithi148 ceph-mon[25818]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:24:44.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:44 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-21T12:24:44.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:44 smithi078 ceph-mon[23681]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:24:44.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:44 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-21T12:24:45.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:45 smithi148 ceph-mon[25818]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:45.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:45 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-21T12:24:45.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:45 smithi148 ceph-mon[25818]: osdmap e60: 8 total, 8 up, 8 in 2024-09-21T12:24:45.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:45 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-21T12:24:45.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:45 smithi078 ceph-mon[23681]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:45.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:45 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-21T12:24:45.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:45 smithi078 ceph-mon[23681]: osdmap e60: 8 total, 8 up, 8 in 2024-09-21T12:24:45.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:45 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-21T12:24:45.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:45 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi078[23677]: 2024-09-21T12:24:45.235+0000 7f975fa63700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T12:24:46.005 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-09-21T12:24:46.015 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-09-21T12:24:46.015 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph orch ls -f json 2024-09-21T12:24:46.517 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:46 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-21T12:24:46.517 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:46 smithi078 ceph-mon[23681]: osdmap e61: 8 total, 8 up, 8 in 2024-09-21T12:24:46.517 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:46 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-21T12:24:46.517 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:46 smithi078 ceph-mon[23681]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T12:24:46.517 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:46 smithi078 ceph-mon[23681]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-21T12:24:46.517 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:46 smithi078 ceph-mon[23681]: osdmap e62: 8 total, 8 up, 8 in 2024-09-21T12:24:46.517 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:46 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-21T12:24:46.517 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:46 smithi078 ceph-mon[23681]: fsmap foofs:0 2024-09-21T12:24:46.518 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:46 smithi078 ceph-mon[23681]: Saving service mds.foofs spec with placement count:2 2024-09-21T12:24:46.518 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:46 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:24:46.518 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:46 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:24:46.518 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:46 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:24:46.518 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:46 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:24:46.518 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:46 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:24:46.518 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:46 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:24:46.518 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:46 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.ytpzwh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T12:24:46.518 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:46 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.ytpzwh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T12:24:46.518 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:46 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:24:46.518 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:46 smithi078 ceph-mon[23681]: Deploying daemon mds.foofs.smithi078.ytpzwh on smithi078 2024-09-21T12:24:46.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:46 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-21T12:24:46.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:46 smithi148 ceph-mon[25818]: osdmap e61: 8 total, 8 up, 8 in 2024-09-21T12:24:46.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:46 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-21T12:24:46.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:46 smithi148 ceph-mon[25818]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T12:24:46.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:46 smithi148 ceph-mon[25818]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-21T12:24:46.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:46 smithi148 ceph-mon[25818]: osdmap e62: 8 total, 8 up, 8 in 2024-09-21T12:24:46.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:46 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-21T12:24:46.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:46 smithi148 ceph-mon[25818]: fsmap foofs:0 2024-09-21T12:24:46.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:46 smithi148 ceph-mon[25818]: Saving service mds.foofs spec with placement count:2 2024-09-21T12:24:46.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:46 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:24:46.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:46 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:24:46.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:46 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:24:46.553 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:46 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:24:46.553 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:46 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:24:46.553 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:46 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:24:46.553 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:46 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.ytpzwh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T12:24:46.553 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:46 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.ytpzwh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T12:24:46.553 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:46 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:24:46.554 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:46 smithi148 ceph-mon[25818]: Deploying daemon mds.foofs.smithi078.ytpzwh on smithi078 2024-09-21T12:24:47.497 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:47 smithi078 ceph-mon[23681]: pgmap v216: 65 pgs: 24 creating+peering, 38 unknown, 3 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:47.497 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:47 smithi078 ceph-mon[23681]: osdmap e63: 8 total, 8 up, 8 in 2024-09-21T12:24:47.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:47 smithi148 ceph-mon[25818]: pgmap v216: 65 pgs: 24 creating+peering, 38 unknown, 3 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:47.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:47 smithi148 ceph-mon[25818]: osdmap e63: 8 total, 8 up, 8 in 2024-09-21T12:24:48.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:48 smithi078 ceph-mon[23681]: osdmap e64: 8 total, 8 up, 8 in 2024-09-21T12:24:48.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:48 smithi148 ceph-mon[25818]: osdmap e64: 8 total, 8 up, 8 in 2024-09-21T12:24:49.052 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T12:24:49.052 INFO:teuthology.orchestra.run.smithi078.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-21T12:18:11.788478Z", "last_refresh": "2024-09-21T12:21:50.309146Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-21T12:18:05.156920Z", "last_refresh": "2024-09-21T12:21:50.309211Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-21T12:18:08.632362Z", "last_refresh": "2024-09-21T12:21:50.309289Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-21T12:24:45.260730Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-21T12:24:45.253345Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-21T12:18:03.699549Z", "last_refresh": "2024-09-21T12:21:50.309075Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi078:172.21.15.78=smithi078", "smithi148:172.21.15.148=smithi148"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-21T12:18:55.548938Z", "last_refresh": "2024-09-21T12:21:50.308925Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-21T12:18:10.264330Z", "last_refresh": "2024-09-21T12:21:50.309347Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-21T12:21:50.309464Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-21T12:18:06.747420Z", "last_refresh": "2024-09-21T12:21:50.309405Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-21T12:24:49.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:49 smithi148 ceph-mon[25818]: pgmap v219: 65 pgs: 29 creating+peering, 14 unknown, 22 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:49.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:49 smithi078 ceph-mon[23681]: pgmap v219: 65 pgs: 29 creating+peering, 14 unknown, 22 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:49.711 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-21T12:24:50.712 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph orch ls -f json 2024-09-21T12:24:51.007 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:50 smithi078 ceph-mon[23681]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T12:24:51.007 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:50 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:24:51.007 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:50 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi148.uukezr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T12:24:51.007 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:50 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi148.uukezr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T12:24:51.007 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:50 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:24:51.007 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:50 smithi078 ceph-mon[23681]: mds.? [v2:172.21.15.78:6834/169294426,v1:172.21.15.78:6835/169294426] up:boot 2024-09-21T12:24:51.007 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:50 smithi078 ceph-mon[23681]: daemon mds.foofs.smithi078.ytpzwh assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-21T12:24:51.007 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:50 smithi078 ceph-mon[23681]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T12:24:51.007 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:50 smithi078 ceph-mon[23681]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-21T12:24:51.007 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:50 smithi078 ceph-mon[23681]: Cluster is now healthy 2024-09-21T12:24:51.008 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:50 smithi078 ceph-mon[23681]: fsmap foofs:0 1 up:standby 2024-09-21T12:24:51.008 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:50 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.ytpzwh"}]: dispatch 2024-09-21T12:24:51.008 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:50 smithi078 ceph-mon[23681]: fsmap foofs:1 {0=foofs.smithi078.ytpzwh=up:creating} 2024-09-21T12:24:51.008 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:50 smithi078 ceph-mon[23681]: daemon mds.foofs.smithi078.ytpzwh is now active in filesystem foofs as rank 0 2024-09-21T12:24:51.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:50 smithi148 ceph-mon[25818]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T12:24:51.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:50 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:24:51.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:50 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi148.uukezr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T12:24:51.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:50 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi148.uukezr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T12:24:51.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:50 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:24:51.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:50 smithi148 ceph-mon[25818]: mds.? [v2:172.21.15.78:6834/169294426,v1:172.21.15.78:6835/169294426] up:boot 2024-09-21T12:24:51.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:50 smithi148 ceph-mon[25818]: daemon mds.foofs.smithi078.ytpzwh assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-21T12:24:51.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:50 smithi148 ceph-mon[25818]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T12:24:51.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:50 smithi148 ceph-mon[25818]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-21T12:24:51.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:50 smithi148 ceph-mon[25818]: Cluster is now healthy 2024-09-21T12:24:51.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:50 smithi148 ceph-mon[25818]: fsmap foofs:0 1 up:standby 2024-09-21T12:24:51.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:50 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.ytpzwh"}]: dispatch 2024-09-21T12:24:51.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:50 smithi148 ceph-mon[25818]: fsmap foofs:1 {0=foofs.smithi078.ytpzwh=up:creating} 2024-09-21T12:24:51.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:50 smithi148 ceph-mon[25818]: daemon mds.foofs.smithi078.ytpzwh is now active in filesystem foofs as rank 0 2024-09-21T12:24:52.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:51 smithi148 ceph-mon[25818]: pgmap v220: 65 pgs: 11 creating+peering, 54 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:52.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:51 smithi148 ceph-mon[25818]: Deploying daemon mds.foofs.smithi148.uukezr on smithi148 2024-09-21T12:24:52.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:51 smithi148 ceph-mon[25818]: mds.? [v2:172.21.15.78:6834/169294426,v1:172.21.15.78:6835/169294426] up:active 2024-09-21T12:24:52.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:51 smithi148 ceph-mon[25818]: fsmap foofs:1 {0=foofs.smithi078.ytpzwh=up:active} 2024-09-21T12:24:52.154 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:51 smithi078 ceph-mon[23681]: pgmap v220: 65 pgs: 11 creating+peering, 54 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:24:52.154 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:51 smithi078 ceph-mon[23681]: Deploying daemon mds.foofs.smithi148.uukezr on smithi148 2024-09-21T12:24:52.154 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:51 smithi078 ceph-mon[23681]: mds.? [v2:172.21.15.78:6834/169294426,v1:172.21.15.78:6835/169294426] up:active 2024-09-21T12:24:52.155 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:51 smithi078 ceph-mon[23681]: fsmap foofs:1 {0=foofs.smithi078.ytpzwh=up:active} 2024-09-21T12:24:52.674 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T12:24:52.674 INFO:teuthology.orchestra.run.smithi078.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-21T12:18:11.788478Z", "last_refresh": "2024-09-21T12:21:50.309146Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-21T12:18:05.156920Z", "last_refresh": "2024-09-21T12:21:50.309211Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-21T12:18:08.632362Z", "last_refresh": "2024-09-21T12:21:50.309289Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-21T12:24:45.260730Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-21T12:24:45.253345Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-21T12:18:03.699549Z", "last_refresh": "2024-09-21T12:21:50.309075Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi078:172.21.15.78=smithi078", "smithi148:172.21.15.148=smithi148"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-21T12:18:55.548938Z", "last_refresh": "2024-09-21T12:21:50.308925Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-21T12:18:10.264330Z", "last_refresh": "2024-09-21T12:21:50.309347Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-21T12:21:50.309464Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-21T12:18:06.747420Z", "last_refresh": "2024-09-21T12:21:50.309405Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-21T12:24:53.493 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-21T12:24:53.781 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:53 smithi078 ceph-mon[23681]: pgmap v221: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 1 op/s 2024-09-21T12:24:53.781 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:53 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:24:53.781 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:53 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:24:53.781 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:53 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:24:53.781 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:53 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:24:53.781 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:53 smithi078 ceph-mon[23681]: from='client.14498 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T12:24:53.781 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:53 smithi078 ceph-mon[23681]: mds.? [v2:172.21.15.148:6832/3237004392,v1:172.21.15.148:6833/3237004392] up:boot 2024-09-21T12:24:53.781 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:53 smithi078 ceph-mon[23681]: fsmap foofs:1 {0=foofs.smithi078.ytpzwh=up:active} 1 up:standby 2024-09-21T12:24:53.781 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:53 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi148.uukezr"}]: dispatch 2024-09-21T12:24:53.781 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:53 smithi078 ceph-mon[23681]: fsmap foofs:1 {0=foofs.smithi078.ytpzwh=up:active} 1 up:standby 2024-09-21T12:24:53.914 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:53 smithi148 ceph-mon[25818]: pgmap v221: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 1 op/s 2024-09-21T12:24:53.914 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:53 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:24:53.914 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:53 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:24:53.914 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:53 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:24:53.914 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:53 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:24:53.914 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:53 smithi148 ceph-mon[25818]: from='client.14498 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T12:24:53.914 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:53 smithi148 ceph-mon[25818]: mds.? [v2:172.21.15.148:6832/3237004392,v1:172.21.15.148:6833/3237004392] up:boot 2024-09-21T12:24:53.914 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:53 smithi148 ceph-mon[25818]: fsmap foofs:1 {0=foofs.smithi078.ytpzwh=up:active} 1 up:standby 2024-09-21T12:24:53.914 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:53 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi148.uukezr"}]: dispatch 2024-09-21T12:24:53.915 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:53 smithi148 ceph-mon[25818]: fsmap foofs:1 {0=foofs.smithi078.ytpzwh=up:active} 1 up:standby 2024-09-21T12:24:54.494 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph orch ls -f json 2024-09-21T12:24:55.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:55 smithi148 ceph-mon[25818]: pgmap v222: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 1 op/s 2024-09-21T12:24:55.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:55 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:24:55.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:55 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:24:55.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:55 smithi078 ceph-mon[23681]: pgmap v222: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 1 op/s 2024-09-21T12:24:55.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:55 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:24:55.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:55 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:24:57.089 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T12:24:57.089 INFO:teuthology.orchestra.run.smithi078.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-21T12:18:11.788478Z", "last_refresh": "2024-09-21T12:21:50.309146Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-21T12:18:05.156920Z", "last_refresh": "2024-09-21T12:21:50.309211Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-21T12:18:08.632362Z", "last_refresh": "2024-09-21T12:21:50.309289Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-21T12:24:45.260730Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-21T12:24:45.253345Z", "last_refresh": "2024-09-21T12:24:54.474590Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-21T12:18:03.699549Z", "last_refresh": "2024-09-21T12:21:50.309075Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi078:172.21.15.78=smithi078", "smithi148:172.21.15.148=smithi148"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-21T12:18:55.548938Z", "last_refresh": "2024-09-21T12:21:50.308925Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-21T12:18:10.264330Z", "last_refresh": "2024-09-21T12:21:50.309347Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-21T12:21:50.309464Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-21T12:18:06.747420Z", "last_refresh": "2024-09-21T12:21:50.309405Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-21T12:24:57.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:57 smithi148 ceph-mon[25818]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-21T12:24:57.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:57 smithi078 ceph-mon[23681]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-21T12:24:57.876 INFO:tasks.cephadm:mds.foofs has 1/2 2024-09-21T12:24:58.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:58 smithi078 ceph-mon[23681]: from='client.14506 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T12:24:58.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:58 smithi148 ceph-mon[25818]: from='client.14506 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T12:24:58.877 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph orch ls -f json 2024-09-21T12:24:59.305 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:59 smithi078 ceph-mon[23681]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-09-21T12:24:59.305 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:24:59 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:24:59.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:59 smithi148 ceph-mon[25818]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-09-21T12:24:59.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:24:59 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:01.289 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T12:25:01.289 INFO:teuthology.orchestra.run.smithi078.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-21T12:18:11.788478Z", "last_refresh": "2024-09-21T12:25:00.630799Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-21T12:18:05.156920Z", "last_refresh": "2024-09-21T12:24:54.473993Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-21T12:18:08.632362Z", "last_refresh": "2024-09-21T12:25:00.630924Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-21T12:24:45.260730Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-21T12:24:45.253345Z", "last_refresh": "2024-09-21T12:24:54.474590Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-21T12:18:03.699549Z", "last_refresh": "2024-09-21T12:24:54.474141Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi078:172.21.15.78=smithi078", "smithi148:172.21.15.148=smithi148"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-21T12:18:55.548938Z", "last_refresh": "2024-09-21T12:24:54.474213Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-21T12:18:10.264330Z", "last_refresh": "2024-09-21T12:24:54.474292Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-21T12:24:54.474353Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-21T12:18:06.747420Z", "last_refresh": "2024-09-21T12:25:00.631063Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-21T12:25:01.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:01 smithi078 ceph-mon[23681]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T12:25:01.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:01 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:01.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:01 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:01.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:01 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:25:01.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:01 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:25:01.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:01 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:25:01.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:01 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:02.050 INFO:tasks.cephadm:mds.foofs has 2/2 2024-09-21T12:25:02.050 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-21T12:25:02.057 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-21T12:25:02.058 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-09-21T12:25:02.058 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:01 smithi148 ceph-mon[25818]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T12:25:02.058 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:01 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:02.058 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:01 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:02.058 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:01 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:25:02.059 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:01 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:25:02.059 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:01 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:25:02.059 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:01 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:03.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:02 smithi148 ceph-mon[25818]: from='client.14510 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T12:25:03.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:02 smithi078 ceph-mon[23681]: from='client.14510 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T12:25:03.952 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:03 smithi148 ceph-mon[25818]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T12:25:04.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:03 smithi078 ceph-mon[23681]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T12:25:05.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:04 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-21T12:25:05.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:04 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-21T12:25:05.686 INFO:teuthology.orchestra.run.smithi078.stdout:NFS Cluster Created Successfully 2024-09-21T12:25:05.915 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:05 smithi078 ceph-mon[23681]: pgmap v227: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 426 B/s wr, 3 op/s 2024-09-21T12:25:05.915 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:05 smithi078 ceph-mon[23681]: from='client.14514 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:25:05.915 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:05 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-21T12:25:05.915 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:05 smithi078 ceph-mon[23681]: osdmap e65: 8 total, 8 up, 8 in 2024-09-21T12:25:05.915 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:05 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-21T12:25:06.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:05 smithi148 ceph-mon[25818]: pgmap v227: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 426 B/s wr, 3 op/s 2024-09-21T12:25:06.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:05 smithi148 ceph-mon[25818]: from='client.14514 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:25:06.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:05 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-21T12:25:06.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:05 smithi148 ceph-mon[25818]: osdmap e65: 8 total, 8 up, 8 in 2024-09-21T12:25:06.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:05 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-21T12:25:06.440 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-09-21T12:25:06.690 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:06 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-21T12:25:06.690 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:06 smithi078 ceph-mon[23681]: osdmap e66: 8 total, 8 up, 8 in 2024-09-21T12:25:06.691 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:06 smithi078 ceph-mon[23681]: Saving service nfs.foo spec with placement count:2 2024-09-21T12:25:06.691 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:06 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:06.691 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:06 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:25:06.691 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:06 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:25:06.691 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:06 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:25:06.691 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:06 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:06.691 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:06 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:06.691 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:06 smithi078 ceph-mon[23681]: Creating key for client.nfs.foo.0.0.smithi078.ydbhru 2024-09-21T12:25:06.691 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:06 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.ydbhru", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-21T12:25:06.691 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:06 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.ydbhru", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-21T12:25:06.691 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:06 smithi078 ceph-mon[23681]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-21T12:25:06.691 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:06 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-21T12:25:06.691 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:06 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T12:25:06.692 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:06 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:25:06.692 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:06 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-21T12:25:06.692 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:06 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T12:25:06.692 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:06 smithi078 ceph-mon[23681]: Rados config object exists: conf-nfs.foo 2024-09-21T12:25:06.692 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:06 smithi078 ceph-mon[23681]: Creating key for client.nfs.foo.0.0.smithi078.ydbhru-rgw 2024-09-21T12:25:06.692 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:06 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.ydbhru-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-21T12:25:06.692 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:06 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.ydbhru-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-21T12:25:06.692 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:06 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:25:06.692 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:06 smithi078 ceph-mon[23681]: Deploying daemon nfs.foo.0.0.smithi078.ydbhru on smithi078 2024-09-21T12:25:07.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:06 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-21T12:25:07.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:06 smithi148 ceph-mon[25818]: osdmap e66: 8 total, 8 up, 8 in 2024-09-21T12:25:07.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:06 smithi148 ceph-mon[25818]: Saving service nfs.foo spec with placement count:2 2024-09-21T12:25:07.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:06 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:07.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:06 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:25:07.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:06 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:25:07.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:06 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:25:07.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:06 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:07.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:06 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:07.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:06 smithi148 ceph-mon[25818]: Creating key for client.nfs.foo.0.0.smithi078.ydbhru 2024-09-21T12:25:07.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:06 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.ydbhru", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-21T12:25:07.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:06 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.ydbhru", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-21T12:25:07.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:06 smithi148 ceph-mon[25818]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-21T12:25:07.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:06 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-21T12:25:07.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:06 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T12:25:07.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:06 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:25:07.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:06 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-21T12:25:07.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:06 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T12:25:07.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:06 smithi148 ceph-mon[25818]: Rados config object exists: conf-nfs.foo 2024-09-21T12:25:07.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:06 smithi148 ceph-mon[25818]: Creating key for client.nfs.foo.0.0.smithi078.ydbhru-rgw 2024-09-21T12:25:07.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:06 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.ydbhru-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-21T12:25:07.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:06 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.ydbhru-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-21T12:25:07.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:06 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:25:07.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:06 smithi148 ceph-mon[25818]: Deploying daemon nfs.foo.0.0.smithi078.ydbhru on smithi078 2024-09-21T12:25:08.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:07 smithi148 ceph-mon[25818]: pgmap v230: 97 pgs: 16 creating+peering, 16 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:25:08.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:07 smithi148 ceph-mon[25818]: osdmap e67: 8 total, 8 up, 8 in 2024-09-21T12:25:08.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:07 smithi148 ceph-mon[25818]: mgrmap e20: smithi078.ntwyan(active, since 5m), standbys: smithi148.oyiqmx 2024-09-21T12:25:08.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:07 smithi078 ceph-mon[23681]: pgmap v230: 97 pgs: 16 creating+peering, 16 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:25:08.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:07 smithi078 ceph-mon[23681]: osdmap e67: 8 total, 8 up, 8 in 2024-09-21T12:25:08.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:07 smithi078 ceph-mon[23681]: mgrmap e20: smithi078.ntwyan(active, since 5m), standbys: smithi148.oyiqmx 2024-09-21T12:25:09.748 INFO:teuthology.orchestra.run.smithi078.stderr:Invalid command: Unexpected argument '--clusterid' 2024-09-21T12:25:09.748 INFO:teuthology.orchestra.run.smithi078.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-09-21T12:25:09.749 INFO:teuthology.orchestra.run.smithi078.stderr:Error EINVAL: invalid command 2024-09-21T12:25:09.920 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:09 smithi078 ceph-mon[23681]: pgmap v232: 97 pgs: 16 creating+peering, 81 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:25:10.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:09 smithi148 ceph-mon[25818]: pgmap v232: 97 pgs: 16 creating+peering, 81 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-21T12:25:10.110 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:25:10.111 INFO:teuthology.orchestra.run.smithi078.stdout: "bind": "/fake", 2024-09-21T12:25:10.111 INFO:teuthology.orchestra.run.smithi078.stdout: "fs": "foofs", 2024-09-21T12:25:10.111 INFO:teuthology.orchestra.run.smithi078.stdout: "path": "/", 2024-09-21T12:25:10.111 INFO:teuthology.orchestra.run.smithi078.stdout: "cluster": "foo", 2024-09-21T12:25:10.111 INFO:teuthology.orchestra.run.smithi078.stdout: "mode": "RW" 2024-09-21T12:25:10.111 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:25:10.874 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-09-21T12:25:11.179 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:11 smithi078 ceph-mon[23681]: pgmap v233: 97 pgs: 5 creating+peering, 92 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-21T12:25:11.180 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:11 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:11.180 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:11 smithi078 ceph-mon[23681]: Creating key for client.nfs.foo.1.0.smithi148.einuzt 2024-09-21T12:25:11.180 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:11 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi148.einuzt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-21T12:25:11.180 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:11 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi148.einuzt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-21T12:25:11.180 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:11 smithi078 ceph-mon[23681]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-21T12:25:11.180 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:11 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-21T12:25:11.180 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:11 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T12:25:11.180 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:11 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:25:11.180 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:11 smithi078 ceph-mon[23681]: from='client.14536 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:25:11.180 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:11 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-21T12:25:11.180 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:11 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-21T12:25:11.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:11 smithi148 ceph-mon[25818]: pgmap v233: 97 pgs: 5 creating+peering, 92 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-21T12:25:11.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:11 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:11.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:11 smithi148 ceph-mon[25818]: Creating key for client.nfs.foo.1.0.smithi148.einuzt 2024-09-21T12:25:11.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:11 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi148.einuzt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-21T12:25:11.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:11 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi148.einuzt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-21T12:25:11.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:11 smithi148 ceph-mon[25818]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-21T12:25:11.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:11 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-21T12:25:11.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:11 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T12:25:11.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:11 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:25:11.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:11 smithi148 ceph-mon[25818]: from='client.14536 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:25:11.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:11 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-21T12:25:11.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:11 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-21T12:25:13.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:13 smithi148 ceph-mon[25818]: pgmap v234: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 138 B/s rd, 277 B/s wr, 0 op/s 2024-09-21T12:25:13.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:13 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-21T12:25:13.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:13 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T12:25:13.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:13 smithi148 ceph-mon[25818]: Rados config object exists: conf-nfs.foo 2024-09-21T12:25:13.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:13 smithi148 ceph-mon[25818]: Creating key for client.nfs.foo.1.0.smithi148.einuzt-rgw 2024-09-21T12:25:13.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:13 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi148.einuzt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-21T12:25:13.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:13 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi148.einuzt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-21T12:25:13.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:13 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:25:13.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:13 smithi148 ceph-mon[25818]: Deploying daemon nfs.foo.1.0.smithi148.einuzt on smithi148 2024-09-21T12:25:13.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:13 smithi148 ceph-mon[25818]: from='client.14552 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:25:13.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:13 smithi078 ceph-mon[23681]: pgmap v234: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 138 B/s rd, 277 B/s wr, 0 op/s 2024-09-21T12:25:13.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:13 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-21T12:25:13.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:13 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T12:25:13.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:13 smithi078 ceph-mon[23681]: Rados config object exists: conf-nfs.foo 2024-09-21T12:25:13.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:13 smithi078 ceph-mon[23681]: Creating key for client.nfs.foo.1.0.smithi148.einuzt-rgw 2024-09-21T12:25:13.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:13 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi148.einuzt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-21T12:25:13.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:13 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi148.einuzt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-21T12:25:13.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:13 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:25:13.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:13 smithi078 ceph-mon[23681]: Deploying daemon nfs.foo.1.0.smithi148.einuzt on smithi148 2024-09-21T12:25:13.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:13 smithi078 ceph-mon[23681]: from='client.14552 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:25:14.357 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:14 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.18", "id": [7, 2]}]: dispatch 2024-09-21T12:25:14.358 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:14 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-09-21T12:25:14.358 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:14 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-09-21T12:25:14.358 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:14 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-09-21T12:25:14.358 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:14 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/mirror_snapshot_schedule"}]: dispatch 2024-09-21T12:25:14.358 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:14 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/trash_purge_schedule"}]: dispatch 2024-09-21T12:25:14.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.18", "id": [7, 2]}]: dispatch 2024-09-21T12:25:14.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-09-21T12:25:14.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-09-21T12:25:14.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-09-21T12:25:14.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/mirror_snapshot_schedule"}]: dispatch 2024-09-21T12:25:14.557 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/trash_purge_schedule"}]: dispatch 2024-09-21T12:25:15.367 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:15 smithi148 ceph-mon[25818]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 122 B/s rd, 244 B/s wr, 0 op/s 2024-09-21T12:25:15.367 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:15 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.18", "id": [7, 2]}]': finished 2024-09-21T12:25:15.367 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:15 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-09-21T12:25:15.367 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:15 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-09-21T12:25:15.367 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:15 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-09-21T12:25:15.367 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:15 smithi148 ceph-mon[25818]: osdmap e68: 8 total, 8 up, 8 in 2024-09-21T12:25:15.367 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:15 smithi148 ceph-mon[25818]: from='client.14566 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:25:15.404 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:15 smithi078 ceph-mon[23681]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 122 B/s rd, 244 B/s wr, 0 op/s 2024-09-21T12:25:15.404 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:15 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.18", "id": [7, 2]}]': finished 2024-09-21T12:25:15.404 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:15 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-09-21T12:25:15.404 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:15 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-09-21T12:25:15.404 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:15 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-09-21T12:25:15.404 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:15 smithi078 ceph-mon[23681]: osdmap e68: 8 total, 8 up, 8 in 2024-09-21T12:25:15.404 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:15 smithi078 ceph-mon[23681]: from='client.14566 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:25:16.174 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:16 smithi078 ceph-mon[23681]: osdmap e69: 8 total, 8 up, 8 in 2024-09-21T12:25:16.174 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:16 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:16.174 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:16 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:25:16.174 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:16 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:25:16.174 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:16 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:25:16.174 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:16 smithi078 ceph-mon[23681]: from='client.14578 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:25:16.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:16 smithi148 ceph-mon[25818]: osdmap e69: 8 total, 8 up, 8 in 2024-09-21T12:25:16.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:16 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:16.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:16 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:25:16.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:16 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:25:16.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:16 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:25:16.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:16 smithi148 ceph-mon[25818]: from='client.14578 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:25:17.640 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:17 smithi078 ceph-mon[23681]: pgmap v238: 97 pgs: 1 peering, 96 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1.9 KiB/s wr, 4 op/s 2024-09-21T12:25:17.640 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:17 smithi078 ceph-mon[23681]: from='client.14582 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:25:17.683 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:17 smithi148 ceph-mon[25818]: pgmap v238: 97 pgs: 1 peering, 96 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1.9 KiB/s wr, 4 op/s 2024-09-21T12:25:17.683 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:17 smithi148 ceph-mon[25818]: from='client.14582 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:25:19.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:19 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:19.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:19 smithi148 ceph-mon[25818]: pgmap v239: 97 pgs: 1 peering, 96 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.6 KiB/s wr, 3 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T12:25:19.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:19 smithi148 ceph-mon[25818]: from='client.14586 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:25:19.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:19 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:19.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:19 smithi078 ceph-mon[23681]: pgmap v239: 97 pgs: 1 peering, 96 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.6 KiB/s wr, 3 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T12:25:19.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:19 smithi078 ceph-mon[23681]: from='client.14586 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:25:20.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:20 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:20.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:20 smithi078 ceph-mon[23681]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:25:20.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:20 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:20.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:20 smithi148 ceph-mon[25818]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:25:20.887 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo ?:2049 2/2 2s ago 15s count:2 2024-09-21T12:25:21.632 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-21T12:25:21.641 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-21T12:25:21.641 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-09-21T12:25:21.668 INFO:teuthology.orchestra.run.smithi078.stderr:+ mkdir /mnt/foo 2024-09-21T12:25:21.690 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-09-21T12:25:21.751 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:21 smithi078 ceph-mon[23681]: pgmap v240: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 KiB/s wr, 8 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T12:25:21.751 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:21 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:21.751 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:21 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:21.751 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:21 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:21.751 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:21 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:25:21.751 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:21 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:25:21.751 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:21 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:25:21.751 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:21 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:21.752 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:21 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:21.752 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:21 smithi078 ceph-mon[23681]: from='client.14594 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:25:21.753 INFO:teuthology.orchestra.run.smithi078.stderr:++ hostname 2024-09-21T12:25:21.754 INFO:teuthology.orchestra.run.smithi078.stderr:+ mount -t nfs smithi078:/fake /mnt/foo -o sync 2024-09-21T12:25:21.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:21 smithi148 ceph-mon[25818]: pgmap v240: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 KiB/s wr, 8 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T12:25:21.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:21 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:21.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:21 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:21.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:21 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:21.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:21 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:25:21.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:21 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:25:21.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:21 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:25:21.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:21 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:21.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:21 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:21.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:21 smithi148 ceph-mon[25818]: from='client.14594 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:25:22.182 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-09-21T12:25:22.248 INFO:teuthology.orchestra.run.smithi078.stderr:+ echo test 2024-09-21T12:25:22.258 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-09-21T12:25:22.321 INFO:teuthology.orchestra.run.smithi078.stderr:+ sync 2024-09-21T12:25:22.589 INFO:teuthology.run_tasks:Running task parallel... 2024-09-21T12:25:22.605 INFO:teuthology.task.parallel:starting parallel... 2024-09-21T12:25:22.605 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-21T12:25:22.606 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-21T12:25:22.606 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-21T12:25:22.606 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -e sha1=441784946648ff53dd310293f9dd752afeeea72a -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-21T12:25:22.607 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-21T12:25:22.607 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-09-21T12:25:22.608 INFO:teuthology.task.exec:Executing custom commands... 2024-09-21T12:25:22.608 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-21T12:25:22.608 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-09-21T12:25:22.659 INFO:teuthology.orchestra.run.smithi078.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-21T12:25:22.659 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T12:25:22.659 INFO:teuthology.orchestra.run.smithi078.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-09-21T12:25:22.661 INFO:teuthology.orchestra.run.smithi078.stdout:3 of 5 processes prepared for launch 0 sec 2024-09-21T12:25:22.761 INFO:teuthology.orchestra.run.smithi078.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-21T12:25:22.761 INFO:teuthology.orchestra.run.smithi078.stdout:releasing clients 2024-09-21T12:25:23.756 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:23 smithi078 ceph-mon[23681]: pgmap v241: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 KiB/s wr, 8 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T12:25:23.761 INFO:teuthology.orchestra.run.smithi078.stdout: 5 141 30.44 MB/sec warmup 1 sec latency 23.648 ms 2024-09-21T12:25:23.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:23 smithi148 ceph-mon[25818]: pgmap v241: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 KiB/s wr, 8 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T12:25:24.761 INFO:teuthology.orchestra.run.smithi078.stdout: 5 246 23.35 MB/sec warmup 2 sec latency 31.797 ms 2024-09-21T12:25:25.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:25 smithi078 ceph-mon[23681]: pgmap v242: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 4.1 KiB/s rd, 2.5 KiB/s wr, 7 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T12:25:25.761 INFO:teuthology.orchestra.run.smithi078.stdout: 5 373 22.11 MB/sec warmup 3 sec latency 30.073 ms 2024-09-21T12:25:25.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:25 smithi148 ceph-mon[25818]: pgmap v242: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 4.1 KiB/s rd, 2.5 KiB/s wr, 7 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T12:25:26.304 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -e sha1=441784946648ff53dd310293f9dd752afeeea72a -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-21T12:25:26.761 INFO:teuthology.orchestra.run.smithi078.stdout: 5 516 22.44 MB/sec warmup 4 sec latency 24.796 ms 2024-09-21T12:25:27.752 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:27 smithi078 ceph-mon[23681]: pgmap v243: 97 pgs: 97 active+clean; 36 MiB data, 73 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 3.4 MiB/s wr, 151 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T12:25:27.761 INFO:teuthology.orchestra.run.smithi078.stdout: 5 756 21.34 MB/sec warmup 5 sec latency 31.620 ms 2024-09-21T12:25:27.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:27 smithi148 ceph-mon[25818]: pgmap v243: 97 pgs: 97 active+clean; 36 MiB data, 73 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 3.4 MiB/s wr, 151 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T12:25:28.761 INFO:teuthology.orchestra.run.smithi078.stdout: 5 1213 19.22 MB/sec warmup 6 sec latency 18.249 ms 2024-09-21T12:25:29.761 INFO:teuthology.orchestra.run.smithi078.stdout: 5 2090 19.04 MB/sec warmup 7 sec latency 23.862 ms 2024-09-21T12:25:29.762 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:29 smithi078 ceph-mon[23681]: pgmap v244: 97 pgs: 97 active+clean; 64 MiB data, 102 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 5.4 MiB/s wr, 181 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T12:25:29.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:29 smithi148 ceph-mon[25818]: pgmap v244: 97 pgs: 97 active+clean; 64 MiB data, 102 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 5.4 MiB/s wr, 181 op/s; 0 B/s, 0 objects/s recovering 2024-09-21T12:25:30.761 INFO:teuthology.orchestra.run.smithi078.stdout: 5 2653 18.83 MB/sec warmup 8 sec latency 38.006 ms 2024-09-21T12:25:30.778 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -e sha1=441784946648ff53dd310293f9dd752afeeea72a -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-21T12:25:31.554 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:31 smithi078 ceph-mon[23681]: pgmap v245: 97 pgs: 97 active+clean; 95 MiB data, 182 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 8.5 MiB/s wr, 462 op/s 2024-09-21T12:25:31.761 INFO:teuthology.orchestra.run.smithi078.stdout: 5 3248 19.37 MB/sec warmup 9 sec latency 63.232 ms 2024-09-21T12:25:31.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:31 smithi148 ceph-mon[25818]: pgmap v245: 97 pgs: 97 active+clean; 95 MiB data, 182 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 8.5 MiB/s wr, 462 op/s 2024-09-21T12:25:32.761 INFO:teuthology.orchestra.run.smithi078.stdout: 5 3740 19.68 MB/sec warmup 10 sec latency 29.109 ms 2024-09-21T12:25:33.599 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:33 smithi078 ceph-mon[23681]: pgmap v246: 97 pgs: 97 active+clean; 116 MiB data, 219 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 11 MiB/s wr, 532 op/s 2024-09-21T12:25:33.762 INFO:teuthology.orchestra.run.smithi078.stdout: 5 3927 18.17 MB/sec warmup 11 sec latency 46.096 ms 2024-09-21T12:25:33.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:33 smithi148 ceph-mon[25818]: pgmap v246: 97 pgs: 97 active+clean; 116 MiB data, 219 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 11 MiB/s wr, 532 op/s 2024-09-21T12:25:34.762 INFO:teuthology.orchestra.run.smithi078.stdout: 5 4462 17.00 MB/sec warmup 12 sec latency 42.979 ms 2024-09-21T12:25:35.509 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -e sha1=441784946648ff53dd310293f9dd752afeeea72a -- bash -c 'ceph mgr module enable nfs --force' 2024-09-21T12:25:35.629 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:35 smithi078 ceph-mon[23681]: pgmap v247: 97 pgs: 97 active+clean; 118 MiB data, 220 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 11 MiB/s wr, 544 op/s 2024-09-21T12:25:35.762 INFO:teuthology.orchestra.run.smithi078.stdout: 5 4827 16.45 MB/sec warmup 13 sec latency 105.470 ms 2024-09-21T12:25:35.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:35 smithi148 ceph-mon[25818]: pgmap v247: 97 pgs: 97 active+clean; 118 MiB data, 220 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 11 MiB/s wr, 544 op/s 2024-09-21T12:25:36.762 INFO:teuthology.orchestra.run.smithi078.stdout: 5 5453 16.29 MB/sec warmup 14 sec latency 41.696 ms 2024-09-21T12:25:37.674 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:37 smithi078 ceph-mon[23681]: pgmap v248: 97 pgs: 97 active+clean; 133 MiB data, 237 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 13 MiB/s wr, 771 op/s 2024-09-21T12:25:37.762 INFO:teuthology.orchestra.run.smithi078.stdout: 5 5911 15.60 MB/sec warmup 15 sec latency 97.813 ms 2024-09-21T12:25:37.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:37 smithi148 ceph-mon[25818]: pgmap v248: 97 pgs: 97 active+clean; 133 MiB data, 237 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 13 MiB/s wr, 771 op/s 2024-09-21T12:25:38.762 INFO:teuthology.orchestra.run.smithi078.stdout: 5 6388 15.93 MB/sec warmup 16 sec latency 43.956 ms 2024-09-21T12:25:39.592 INFO:teuthology.orchestra.run.smithi078.stderr:module 'nfs' is already enabled 2024-09-21T12:25:39.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:39 smithi078 ceph-mon[23681]: pgmap v249: 97 pgs: 97 active+clean; 134 MiB data, 244 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 11 MiB/s wr, 673 op/s 2024-09-21T12:25:39.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:39 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/2527773013' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-21T12:25:39.762 INFO:teuthology.orchestra.run.smithi078.stdout: 5 6900 16.09 MB/sec warmup 17 sec latency 77.479 ms 2024-09-21T12:25:39.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:39 smithi148 ceph-mon[25818]: pgmap v249: 97 pgs: 97 active+clean; 134 MiB data, 244 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 11 MiB/s wr, 673 op/s 2024-09-21T12:25:39.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:39 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/2527773013' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-21T12:25:40.762 INFO:teuthology.orchestra.run.smithi078.stdout: 5 7285 16.17 MB/sec warmup 18 sec latency 53.648 ms 2024-09-21T12:25:40.769 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:40 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/2527773013' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-21T12:25:40.769 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:40 smithi078 ceph-mon[23681]: mgrmap e21: smithi078.ntwyan(active, since 6m), standbys: smithi148.oyiqmx 2024-09-21T12:25:40.770 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -e sha1=441784946648ff53dd310293f9dd752afeeea72a -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-21T12:25:41.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:40 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/2527773013' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-21T12:25:41.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:40 smithi148 ceph-mon[25818]: mgrmap e21: smithi078.ntwyan(active, since 6m), standbys: smithi148.oyiqmx 2024-09-21T12:25:41.762 INFO:teuthology.orchestra.run.smithi078.stdout: 5 7489 15.54 MB/sec warmup 19 sec latency 50.222 ms 2024-09-21T12:25:41.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:41 smithi078 ceph-mon[23681]: pgmap v250: 97 pgs: 97 active+clean; 139 MiB data, 290 MiB used, 715 GiB / 715 GiB avail; 8.1 KiB/s rd, 10 MiB/s wr, 847 op/s 2024-09-21T12:25:42.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:41 smithi148 ceph-mon[25818]: pgmap v250: 97 pgs: 97 active+clean; 139 MiB data, 290 MiB used, 715 GiB / 715 GiB avail; 8.1 KiB/s rd, 10 MiB/s wr, 847 op/s 2024-09-21T12:25:42.762 INFO:teuthology.orchestra.run.smithi078.stdout: 5 7890 14.90 MB/sec warmup 20 sec latency 70.373 ms 2024-09-21T12:25:43.600 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:43 smithi078 ceph-mon[23681]: pgmap v251: 97 pgs: 97 active+clean; 139 MiB data, 322 MiB used, 715 GiB / 715 GiB avail; 7.7 KiB/s rd, 8.3 MiB/s wr, 618 op/s 2024-09-21T12:25:43.762 INFO:teuthology.orchestra.run.smithi078.stdout: 5 8234 14.67 MB/sec warmup 21 sec latency 109.721 ms 2024-09-21T12:25:44.039 INFO:teuthology.orchestra.run.smithi078.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:441784946648ff53dd310293f9dd752afeeea72a 2024-09-21T12:25:44.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:43 smithi148 ceph-mon[25818]: pgmap v251: 97 pgs: 97 active+clean; 139 MiB data, 322 MiB used, 715 GiB / 715 GiB avail; 7.7 KiB/s rd, 8.3 MiB/s wr, 618 op/s 2024-09-21T12:25:44.762 INFO:teuthology.orchestra.run.smithi078.stdout: 5 8739 14.64 MB/sec warmup 22 sec latency 102.227 ms 2024-09-21T12:25:45.179 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:45 smithi078 ceph-mon[23681]: from='client.14614 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:441784946648ff53dd310293f9dd752afeeea72a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:25:45.179 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:45 smithi078 ceph-mon[23681]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:441784946648ff53dd310293f9dd752afeeea72a 2024-09-21T12:25:45.179 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:45 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:45.179 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:45 smithi078 ceph-mon[23681]: pgmap v252: 97 pgs: 97 active+clean; 140 MiB data, 322 MiB used, 715 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.3 MiB/s wr, 553 op/s 2024-09-21T12:25:45.179 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:45 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:25:45.179 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:45 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:25:45.179 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:45 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:25:45.179 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:45 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:45.179 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:45 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:45.179 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:45 smithi078 ceph-mon[23681]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:441784946648ff53dd310293f9dd752afeeea72a 2024-09-21T12:25:45.267 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-21T12:25:45.268 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-21T12:25:45.268 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -e sha1=441784946648ff53dd310293f9dd752afeeea72a -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-09-21T12:25:45.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:45 smithi148 ceph-mon[25818]: from='client.14614 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:441784946648ff53dd310293f9dd752afeeea72a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:25:45.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:45 smithi148 ceph-mon[25818]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:441784946648ff53dd310293f9dd752afeeea72a 2024-09-21T12:25:45.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:45 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:45.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:45 smithi148 ceph-mon[25818]: pgmap v252: 97 pgs: 97 active+clean; 140 MiB data, 322 MiB used, 715 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.3 MiB/s wr, 553 op/s 2024-09-21T12:25:45.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:45 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:25:45.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:45 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:25:45.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:45 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:25:45.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:45 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:45.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:45 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:25:45.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:45 smithi148 ceph-mon[25818]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:441784946648ff53dd310293f9dd752afeeea72a 2024-09-21T12:25:45.762 INFO:teuthology.orchestra.run.smithi078.stdout: 5 9188 14.14 MB/sec warmup 23 sec latency 60.205 ms 2024-09-21T12:25:46.762 INFO:teuthology.orchestra.run.smithi078.stdout: 5 9498 13.82 MB/sec warmup 24 sec latency 65.789 ms 2024-09-21T12:25:47.278 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:47 smithi078 ceph-mon[23681]: pgmap v253: 97 pgs: 97 active+clean; 142 MiB data, 396 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 751 op/s 2024-09-21T12:25:47.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:47 smithi148 ceph-mon[25818]: pgmap v253: 97 pgs: 97 active+clean; 142 MiB data, 396 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 751 op/s 2024-09-21T12:25:47.762 INFO:teuthology.orchestra.run.smithi078.stdout: 5 9970 14.18 MB/sec warmup 25 sec latency 41.820 ms 2024-09-21T12:25:48.541 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-21T12:25:48.762 INFO:teuthology.orchestra.run.smithi078.stdout: 5 10534 14.40 MB/sec warmup 26 sec latency 30.409 ms 2024-09-21T12:25:49.291 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:49 smithi078 ceph-mon[23681]: pgmap v254: 97 pgs: 97 active+clean; 137 MiB data, 420 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.7 MiB/s wr, 558 op/s 2024-09-21T12:25:49.292 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:49 smithi078 ceph-mon[23681]: from='client.14618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:25:49.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:49 smithi148 ceph-mon[25818]: pgmap v254: 97 pgs: 97 active+clean; 137 MiB data, 420 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.7 MiB/s wr, 558 op/s 2024-09-21T12:25:49.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:49 smithi148 ceph-mon[25818]: from='client.14618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:25:49.554 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T12:25:49.554 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (5m) 29s ago 7m 23.4M - ba2b418f427c 913ca01a7223 2024-09-21T12:25:49.555 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (6m) 29s ago 6m 6400k - 17.2.0 e1d6a67b021e 4a209e5da5ce 2024-09-21T12:25:49.555 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi148 smithi148 running (6m) 31s ago 6m 6405k - 17.2.0 e1d6a67b021e 0fa5e612307f 2024-09-21T12:25:49.555 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (5m) 29s ago 6m 51.1M - 8.3.5 dad864ee21e9 b6121d08c85b 2024-09-21T12:25:49.555 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.ytpzwh smithi078 running (60s) 29s ago 59s 14.8M - 17.2.0 e1d6a67b021e 5e9b109502a2 2024-09-21T12:25:49.555 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi148.uukezr smithi148 running (57s) 31s ago 56s 11.0M - 17.2.0 e1d6a67b021e b7e8aa0f3a6c 2024-09-21T12:25:49.555 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.ntwyan smithi078 *:9283 running (8m) 29s ago 8m 459M - 17.2.0 e1d6a67b021e caeba9628014 2024-09-21T12:25:49.555 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi148.oyiqmx smithi148 *:8443,9283 running (6m) 31s ago 6m 412M - 17.2.0 e1d6a67b021e 47b925ccff59 2024-09-21T12:25:49.555 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (8m) 29s ago 8m 46.5M 2048M 17.2.0 e1d6a67b021e 8261b0b2c7e0 2024-09-21T12:25:49.555 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi148 smithi148 running (6m) 31s ago 6m 37.7M 2048M 17.2.0 e1d6a67b021e fda961db230f 2024-09-21T12:25:49.555 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.ydbhru smithi078 *:2049 running (40s) 29s ago 39s 49.4M - 4.0 e1d6a67b021e 7e5b0274ac38 2024-09-21T12:25:49.555 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi148.einuzt smithi148 *:2049 running (34s) 31s ago 34s 50.7M - 4.0 e1d6a67b021e 76a8b0eabede 2024-09-21T12:25:49.555 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (6m) 29s ago 6m 16.5M - 1dbe0e931976 e7ca2ce65161 2024-09-21T12:25:49.555 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi148 smithi148 *:9100 running (6m) 31s ago 6m 17.4M - 1dbe0e931976 1474152580ab 2024-09-21T12:25:49.556 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (5m) 29s ago 5m 43.4M 3216M 17.2.0 e1d6a67b021e d4d5e9977070 2024-09-21T12:25:49.556 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (4m) 29s ago 4m 45.5M 3216M 17.2.0 e1d6a67b021e b9a4b952233b 2024-09-21T12:25:49.556 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (4m) 29s ago 4m 40.4M 3216M 17.2.0 e1d6a67b021e 829fb8c2d827 2024-09-21T12:25:49.556 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (4m) 29s ago 4m 40.7M 3216M 17.2.0 e1d6a67b021e ef942c8556d0 2024-09-21T12:25:49.556 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi148 running (3m) 31s ago 3m 41.0M 3984M 17.2.0 e1d6a67b021e 62eefdd572c0 2024-09-21T12:25:49.556 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi148 running (3m) 31s ago 3m 38.9M 3984M 17.2.0 e1d6a67b021e 7b44be78baca 2024-09-21T12:25:49.556 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi148 running (2m) 31s ago 2m 39.9M 3984M 17.2.0 e1d6a67b021e 3d23057910ea 2024-09-21T12:25:49.556 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi148 running (2m) 31s ago 2m 39.7M 3984M 17.2.0 e1d6a67b021e f9e4db9d35fa 2024-09-21T12:25:49.556 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (5m) 29s ago 5m 61.4M - 514e6a882f6e cdcd63cf2477 2024-09-21T12:25:49.762 INFO:teuthology.orchestra.run.smithi078.stdout: 5 10963 14.47 MB/sec warmup 27 sec latency 36.176 ms 2024-09-21T12:25:50.146 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:25:50.146 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-21T12:25:50.146 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:25:50.146 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:25:50.146 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-21T12:25:50.146 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:25:50.146 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:25:50.146 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-21T12:25:50.146 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T12:25:50.146 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:25:50.147 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-21T12:25:50.147 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:25:50.147 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:25:50.147 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-21T12:25:50.147 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-21T12:25:50.147 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T12:25:50.147 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:25:50.304 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:50 smithi078 ceph-mon[23681]: from='client.24367 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:25:50.304 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:50 smithi078 ceph-mon[23681]: from='client.24371 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:25:50.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:50 smithi148 ceph-mon[25818]: from='client.24367 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:25:50.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:50 smithi148 ceph-mon[25818]: from='client.24371 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:25:50.634 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:25:50.634 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:441784946648ff53dd310293f9dd752afeeea72a", 2024-09-21T12:25:50.634 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-21T12:25:50.634 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-09-21T12:25:50.634 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "", 2024-09-21T12:25:50.634 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:441784946648ff53dd310293f9dd752afeeea72a image" 2024-09-21T12:25:50.635 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:25:50.762 INFO:teuthology.orchestra.run.smithi078.stdout: 5 11412 14.05 MB/sec warmup 28 sec latency 56.637 ms 2024-09-21T12:25:51.258 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-21T12:25:51.337 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:51 smithi078 ceph-mon[23681]: pgmap v255: 97 pgs: 97 active+clean; 142 MiB data, 416 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 6.1 MiB/s wr, 610 op/s 2024-09-21T12:25:51.337 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:51 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/135467706' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T12:25:51.337 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:51 smithi078 ceph-mon[23681]: from='client.24379 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:25:51.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:51 smithi148 ceph-mon[25818]: pgmap v255: 97 pgs: 97 active+clean; 142 MiB data, 416 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 6.1 MiB/s wr, 610 op/s 2024-09-21T12:25:51.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:51 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/135467706' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T12:25:51.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:51 smithi148 ceph-mon[25818]: from='client.24379 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:25:51.763 INFO:teuthology.orchestra.run.smithi078.stdout: 5 11659 13.75 MB/sec warmup 29 sec latency 48.864 ms 2024-09-21T12:25:52.349 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:52 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/3521536539' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T12:25:52.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:52 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/3521536539' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T12:25:52.762 INFO:teuthology.orchestra.run.smithi078.stdout: 5 11977 13.69 MB/sec warmup 30 sec latency 41.844 ms 2024-09-21T12:25:53.369 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:53 smithi078 ceph-mon[23681]: pgmap v256: 97 pgs: 97 active+clean; 158 MiB data, 436 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.4 MiB/s wr, 577 op/s 2024-09-21T12:25:53.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:53 smithi148 ceph-mon[25818]: pgmap v256: 97 pgs: 97 active+clean; 158 MiB data, 436 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.4 MiB/s wr, 577 op/s 2024-09-21T12:25:53.763 INFO:teuthology.orchestra.run.smithi078.stdout: 5 12435 13.49 MB/sec warmup 31 sec latency 46.543 ms 2024-09-21T12:25:54.763 INFO:teuthology.orchestra.run.smithi078.stdout: 5 12846 13.22 MB/sec warmup 32 sec latency 50.846 ms 2024-09-21T12:25:55.165 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:55 smithi078 ceph-mon[23681]: pgmap v257: 97 pgs: 97 active+clean; 159 MiB data, 435 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.5 MiB/s wr, 536 op/s 2024-09-21T12:25:55.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:55 smithi148 ceph-mon[25818]: pgmap v257: 97 pgs: 97 active+clean; 159 MiB data, 435 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.5 MiB/s wr, 536 op/s 2024-09-21T12:25:55.763 INFO:teuthology.orchestra.run.smithi078.stdout: 5 13223 13.13 MB/sec warmup 33 sec latency 41.471 ms 2024-09-21T12:25:56.763 INFO:teuthology.orchestra.run.smithi078.stdout: 5 13752 13.44 MB/sec warmup 34 sec latency 40.780 ms 2024-09-21T12:25:57.234 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:57 smithi078 ceph-mon[23681]: pgmap v258: 97 pgs: 97 active+clean; 158 MiB data, 467 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 6.4 MiB/s wr, 687 op/s 2024-09-21T12:25:57.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:57 smithi148 ceph-mon[25818]: pgmap v258: 97 pgs: 97 active+clean; 158 MiB data, 467 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 6.4 MiB/s wr, 687 op/s 2024-09-21T12:25:57.763 INFO:teuthology.orchestra.run.smithi078.stdout: 5 14222 13.55 MB/sec warmup 35 sec latency 68.940 ms 2024-09-21T12:25:58.763 INFO:teuthology.orchestra.run.smithi078.stdout: 5 14565 13.51 MB/sec warmup 36 sec latency 58.387 ms 2024-09-21T12:25:59.254 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:25:59 smithi078 ceph-mon[23681]: pgmap v259: 97 pgs: 97 active+clean; 154 MiB data, 491 MiB used, 715 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.5 MiB/s wr, 519 op/s 2024-09-21T12:25:59.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:25:59 smithi148 ceph-mon[25818]: pgmap v259: 97 pgs: 97 active+clean; 154 MiB data, 491 MiB used, 715 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.5 MiB/s wr, 519 op/s 2024-09-21T12:25:59.763 INFO:teuthology.orchestra.run.smithi078.stdout: 5 15019 13.24 MB/sec warmup 37 sec latency 41.042 ms 2024-09-21T12:26:00.763 INFO:teuthology.orchestra.run.smithi078.stdout: 5 15261 13.02 MB/sec warmup 38 sec latency 49.221 ms 2024-09-21T12:26:01.314 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:01 smithi078 ceph-mon[23681]: pgmap v260: 97 pgs: 97 active+clean; 160 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.2 MiB/s wr, 536 op/s 2024-09-21T12:26:01.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:01 smithi148 ceph-mon[25818]: pgmap v260: 97 pgs: 97 active+clean; 160 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.2 MiB/s wr, 536 op/s 2024-09-21T12:26:01.763 INFO:teuthology.orchestra.run.smithi078.stdout: 5 15502 12.98 MB/sec warmup 39 sec latency 108.107 ms 2024-09-21T12:26:02.763 INFO:teuthology.orchestra.run.smithi078.stdout: 5 15833 12.76 MB/sec warmup 40 sec latency 50.482 ms 2024-09-21T12:26:03.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:03 smithi078 ceph-mon[23681]: pgmap v261: 97 pgs: 97 active+clean; 171 MiB data, 556 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.6 MiB/s wr, 627 op/s 2024-09-21T12:26:03.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:03 smithi148 ceph-mon[25818]: pgmap v261: 97 pgs: 97 active+clean; 171 MiB data, 556 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.6 MiB/s wr, 627 op/s 2024-09-21T12:26:03.763 INFO:teuthology.orchestra.run.smithi078.stdout: 5 16241 12.59 MB/sec warmup 41 sec latency 68.319 ms 2024-09-21T12:26:04.763 INFO:teuthology.orchestra.run.smithi078.stdout: 5 16635 12.45 MB/sec warmup 42 sec latency 51.378 ms 2024-09-21T12:26:05.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:05 smithi078 ceph-mon[23681]: pgmap v262: 97 pgs: 97 active+clean; 170 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.8 MiB/s wr, 448 op/s 2024-09-21T12:26:05.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:05 smithi148 ceph-mon[25818]: pgmap v262: 97 pgs: 97 active+clean; 170 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.8 MiB/s wr, 448 op/s 2024-09-21T12:26:05.763 INFO:teuthology.orchestra.run.smithi078.stdout: 5 17128 12.68 MB/sec warmup 43 sec latency 33.877 ms 2024-09-21T12:26:06.763 INFO:teuthology.orchestra.run.smithi078.stdout: 5 17627 12.89 MB/sec warmup 44 sec latency 58.780 ms 2024-09-21T12:26:07.444 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:07 smithi078 ceph-mon[23681]: pgmap v263: 97 pgs: 97 active+clean; 168 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 5.7 MiB/s wr, 577 op/s 2024-09-21T12:26:07.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:07 smithi148 ceph-mon[25818]: pgmap v263: 97 pgs: 97 active+clean; 168 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 5.7 MiB/s wr, 577 op/s 2024-09-21T12:26:07.763 INFO:teuthology.orchestra.run.smithi078.stdout: 5 17932 12.86 MB/sec warmup 45 sec latency 52.931 ms 2024-09-21T12:26:08.763 INFO:teuthology.orchestra.run.smithi078.stdout: 5 18134 12.66 MB/sec warmup 46 sec latency 71.896 ms 2024-09-21T12:26:09.221 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:09 smithi078 ceph-mon[23681]: pgmap v264: 97 pgs: 97 active+clean; 172 MiB data, 615 MiB used, 715 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.7 MiB/s wr, 550 op/s 2024-09-21T12:26:09.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:09 smithi148 ceph-mon[25818]: pgmap v264: 97 pgs: 97 active+clean; 172 MiB data, 615 MiB used, 715 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.7 MiB/s wr, 550 op/s 2024-09-21T12:26:09.763 INFO:teuthology.orchestra.run.smithi078.stdout: 5 18396 12.43 MB/sec warmup 47 sec latency 94.190 ms 2024-09-21T12:26:10.763 INFO:teuthology.orchestra.run.smithi078.stdout: 5 18669 12.26 MB/sec warmup 48 sec latency 56.259 ms 2024-09-21T12:26:11.295 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:11 smithi078 ceph-mon[23681]: pgmap v265: 97 pgs: 97 active+clean; 177 MiB data, 606 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.9 MiB/s wr, 539 op/s 2024-09-21T12:26:11.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:11 smithi148 ceph-mon[25818]: pgmap v265: 97 pgs: 97 active+clean; 177 MiB data, 606 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.9 MiB/s wr, 539 op/s 2024-09-21T12:26:11.764 INFO:teuthology.orchestra.run.smithi078.stdout: 5 18887 12.07 MB/sec warmup 49 sec latency 70.084 ms 2024-09-21T12:26:12.764 INFO:teuthology.orchestra.run.smithi078.stdout: 5 19309 12.12 MB/sec warmup 50 sec latency 73.448 ms 2024-09-21T12:26:13.318 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:13 smithi078 ceph-mon[23681]: pgmap v266: 97 pgs: 97 active+clean; 186 MiB data, 639 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.0 MiB/s wr, 618 op/s 2024-09-21T12:26:13.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:13 smithi148 ceph-mon[25818]: pgmap v266: 97 pgs: 97 active+clean; 186 MiB data, 639 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.0 MiB/s wr, 618 op/s 2024-09-21T12:26:13.764 INFO:teuthology.orchestra.run.smithi078.stdout: 5 19823 12.00 MB/sec warmup 51 sec latency 53.941 ms 2024-09-21T12:26:14.332 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/mirror_snapshot_schedule"}]: dispatch 2024-09-21T12:26:14.332 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:14 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/trash_purge_schedule"}]: dispatch 2024-09-21T12:26:14.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:14 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/mirror_snapshot_schedule"}]: dispatch 2024-09-21T12:26:14.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:14 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/trash_purge_schedule"}]: dispatch 2024-09-21T12:26:14.764 INFO:teuthology.orchestra.run.smithi078.stdout: 5 20199 11.90 MB/sec warmup 52 sec latency 99.125 ms 2024-09-21T12:26:15.345 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:15 smithi078 ceph-mon[23681]: pgmap v267: 97 pgs: 97 active+clean; 185 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 443 op/s 2024-09-21T12:26:15.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:15 smithi148 ceph-mon[25818]: pgmap v267: 97 pgs: 97 active+clean; 185 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 443 op/s 2024-09-21T12:26:15.764 INFO:teuthology.orchestra.run.smithi078.stdout: 5 20662 12.03 MB/sec warmup 53 sec latency 60.098 ms 2024-09-21T12:26:16.764 INFO:teuthology.orchestra.run.smithi078.stdout: 5 21143 12.24 MB/sec warmup 54 sec latency 45.244 ms 2024-09-21T12:26:17.323 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:17 smithi078 ceph-mon[23681]: pgmap v268: 97 pgs: 97 active+clean; 179 MiB data, 699 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 5.3 MiB/s wr, 531 op/s 2024-09-21T12:26:17.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:17 smithi148 ceph-mon[25818]: pgmap v268: 97 pgs: 97 active+clean; 179 MiB data, 699 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 5.3 MiB/s wr, 531 op/s 2024-09-21T12:26:17.764 INFO:teuthology.orchestra.run.smithi078.stdout: 5 21387 12.15 MB/sec warmup 55 sec latency 68.053 ms 2024-09-21T12:26:18.764 INFO:teuthology.orchestra.run.smithi078.stdout: 5 21601 12.06 MB/sec warmup 56 sec latency 56.046 ms 2024-09-21T12:26:19.386 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:19 smithi078 ceph-mon[23681]: pgmap v269: 97 pgs: 97 active+clean; 182 MiB data, 724 MiB used, 715 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.2 MiB/s wr, 486 op/s 2024-09-21T12:26:19.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:19 smithi148 ceph-mon[25818]: pgmap v269: 97 pgs: 97 active+clean; 182 MiB data, 724 MiB used, 715 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.2 MiB/s wr, 486 op/s 2024-09-21T12:26:19.764 INFO:teuthology.orchestra.run.smithi078.stdout: 5 21791 11.90 MB/sec warmup 57 sec latency 77.681 ms 2024-09-21T12:26:20.764 INFO:teuthology.orchestra.run.smithi078.stdout: 5 22035 11.74 MB/sec warmup 58 sec latency 78.831 ms 2024-09-21T12:26:21.660 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-21T12:26:21.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:21 smithi078 ceph-mon[23681]: pgmap v270: 97 pgs: 97 active+clean; 193 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.5 MiB/s wr, 503 op/s 2024-09-21T12:26:21.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:21 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:26:21.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:21 smithi078 ceph-mon[23681]: Upgrade: Target is version 19.3.0-5104-g44178494 (unknown) 2024-09-21T12:26:21.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:21 smithi078 ceph-mon[23681]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5'] 2024-09-21T12:26:21.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:21 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:26:21.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:21 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:26:21.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:21 smithi078 ceph-mon[23681]: Upgrade: Need to upgrade myself (mgr.smithi078.ntwyan) 2024-09-21T12:26:21.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:21 smithi078 ceph-mon[23681]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 on smithi148 2024-09-21T12:26:21.764 INFO:teuthology.orchestra.run.smithi078.stdout: 5 22299 11.61 MB/sec warmup 59 sec latency 51.863 ms 2024-09-21T12:26:21.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:21 smithi148 ceph-mon[25818]: pgmap v270: 97 pgs: 97 active+clean; 193 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.5 MiB/s wr, 503 op/s 2024-09-21T12:26:21.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:21 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:26:21.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:21 smithi148 ceph-mon[25818]: Upgrade: Target is version 19.3.0-5104-g44178494 (unknown) 2024-09-21T12:26:21.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:21 smithi148 ceph-mon[25818]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, digests ['quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5'] 2024-09-21T12:26:21.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:21 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:26:21.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:21 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:26:21.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:21 smithi148 ceph-mon[25818]: Upgrade: Need to upgrade myself (mgr.smithi078.ntwyan) 2024-09-21T12:26:21.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:21 smithi148 ceph-mon[25818]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 on smithi148 2024-09-21T12:26:22.396 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T12:26:22.396 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (6m) 61s ago 7m 23.4M - ba2b418f427c 913ca01a7223 2024-09-21T12:26:22.396 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (7m) 61s ago 7m 6400k - 17.2.0 e1d6a67b021e 4a209e5da5ce 2024-09-21T12:26:22.396 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi148 smithi148 running (6m) 64s ago 6m 6405k - 17.2.0 e1d6a67b021e 0fa5e612307f 2024-09-21T12:26:22.396 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (5m) 61s ago 7m 51.1M - 8.3.5 dad864ee21e9 b6121d08c85b 2024-09-21T12:26:22.396 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.ytpzwh smithi078 running (93s) 61s ago 92s 14.8M - 17.2.0 e1d6a67b021e 5e9b109502a2 2024-09-21T12:26:22.396 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi148.uukezr smithi148 running (90s) 64s ago 89s 11.0M - 17.2.0 e1d6a67b021e b7e8aa0f3a6c 2024-09-21T12:26:22.396 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.ntwyan smithi078 *:9283 running (8m) 61s ago 8m 459M - 17.2.0 e1d6a67b021e caeba9628014 2024-09-21T12:26:22.396 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi148.oyiqmx smithi148 *:8443,9283 running (6m) 64s ago 6m 412M - 17.2.0 e1d6a67b021e 47b925ccff59 2024-09-21T12:26:22.396 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (8m) 61s ago 8m 46.5M 2048M 17.2.0 e1d6a67b021e 8261b0b2c7e0 2024-09-21T12:26:22.397 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi148 smithi148 running (6m) 64s ago 6m 37.7M 2048M 17.2.0 e1d6a67b021e fda961db230f 2024-09-21T12:26:22.397 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.ydbhru smithi078 *:2049 running (73s) 61s ago 72s 49.4M - 4.0 e1d6a67b021e 7e5b0274ac38 2024-09-21T12:26:22.397 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi148.einuzt smithi148 *:2049 running (67s) 64s ago 67s 50.7M - 4.0 e1d6a67b021e 76a8b0eabede 2024-09-21T12:26:22.397 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (7m) 61s ago 7m 16.5M - 1dbe0e931976 e7ca2ce65161 2024-09-21T12:26:22.397 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi148 smithi148 *:9100 running (6m) 64s ago 6m 17.4M - 1dbe0e931976 1474152580ab 2024-09-21T12:26:22.397 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (6m) 61s ago 6m 43.4M 3216M 17.2.0 e1d6a67b021e d4d5e9977070 2024-09-21T12:26:22.397 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (5m) 61s ago 5m 45.5M 3216M 17.2.0 e1d6a67b021e b9a4b952233b 2024-09-21T12:26:22.397 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (5m) 61s ago 5m 40.4M 3216M 17.2.0 e1d6a67b021e 829fb8c2d827 2024-09-21T12:26:22.397 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (4m) 61s ago 4m 40.7M 3216M 17.2.0 e1d6a67b021e ef942c8556d0 2024-09-21T12:26:22.397 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi148 running (4m) 64s ago 4m 41.0M 3984M 17.2.0 e1d6a67b021e 62eefdd572c0 2024-09-21T12:26:22.397 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi148 running (3m) 64s ago 3m 38.9M 3984M 17.2.0 e1d6a67b021e 7b44be78baca 2024-09-21T12:26:22.397 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi148 running (3m) 64s ago 3m 39.9M 3984M 17.2.0 e1d6a67b021e 3d23057910ea 2024-09-21T12:26:22.397 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi148 running (3m) 64s ago 3m 39.7M 3984M 17.2.0 e1d6a67b021e f9e4db9d35fa 2024-09-21T12:26:22.397 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (6m) 61s ago 6m 61.4M - 514e6a882f6e cdcd63cf2477 2024-09-21T12:26:22.756 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:22 smithi078 ceph-mon[23681]: from='client.14642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:26:22.764 INFO:teuthology.orchestra.run.smithi078.stdout: 5 22549 11.50 MB/sec warmup 60 sec latency 150.751 ms 2024-09-21T12:26:22.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:22 smithi148 ceph-mon[25818]: from='client.14642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:26:22.858 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:26:22.858 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-21T12:26:22.858 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:26:22.858 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:26:22.858 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-21T12:26:22.858 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:26:22.858 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:26:22.858 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-21T12:26:22.858 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T12:26:22.858 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:26:22.858 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-21T12:26:22.859 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:26:22.859 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:26:22.859 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-21T12:26:22.859 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-21T12:26:22.859 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T12:26:22.859 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:26:23.313 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:26:23.313 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5", 2024-09-21T12:26:23.313 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-21T12:26:23.313 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-09-21T12:26:23.313 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "0/23 daemons upgraded", 2024-09-21T12:26:23.313 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 image on host smithi148" 2024-09-21T12:26:23.313 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:26:23.764 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:23 smithi078 ceph-mon[23681]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:26:23.764 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:23 smithi078 ceph-mon[23681]: pgmap v271: 97 pgs: 97 active+clean; 196 MiB data, 746 MiB used, 714 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.2 MiB/s wr, 493 op/s 2024-09-21T12:26:23.764 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:23 smithi078 ceph-mon[23681]: from='client.14650 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:26:23.764 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:23 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/620247471' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T12:26:23.764 INFO:teuthology.orchestra.run.smithi078.stdout: 5 23141 11.54 MB/sec warmup 61 sec latency 90.693 ms 2024-09-21T12:26:23.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:23 smithi148 ceph-mon[25818]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:26:23.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:23 smithi148 ceph-mon[25818]: pgmap v271: 97 pgs: 97 active+clean; 196 MiB data, 746 MiB used, 714 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.2 MiB/s wr, 493 op/s 2024-09-21T12:26:23.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:23 smithi148 ceph-mon[25818]: from='client.14650 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:26:23.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:23 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/620247471' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T12:26:23.930 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-21T12:26:24.527 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:24 smithi078 ceph-mon[23681]: from='client.14658 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:26:24.528 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:24 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/2086706160' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T12:26:24.764 INFO:teuthology.orchestra.run.smithi078.stdout: 5 24095 11.78 MB/sec warmup 62 sec latency 33.464 ms 2024-09-21T12:26:24.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:24 smithi148 ceph-mon[25818]: from='client.14658 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:26:24.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:24 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/2086706160' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T12:26:25.534 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:25 smithi078 ceph-mon[23681]: pgmap v272: 97 pgs: 97 active+clean; 194 MiB data, 747 MiB used, 714 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.1 MiB/s wr, 368 op/s 2024-09-21T12:26:25.764 INFO:teuthology.orchestra.run.smithi078.stdout: 5 24988 12.13 MB/sec warmup 63 sec latency 13.529 ms 2024-09-21T12:26:25.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:25 smithi148 ceph-mon[25818]: pgmap v272: 97 pgs: 97 active+clean; 194 MiB data, 747 MiB used, 714 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.1 MiB/s wr, 368 op/s 2024-09-21T12:26:26.764 INFO:teuthology.orchestra.run.smithi078.stdout: 5 25725 12.17 MB/sec warmup 64 sec latency 16.351 ms 2024-09-21T12:26:27.543 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:27 smithi078 ceph-mon[23681]: pgmap v273: 97 pgs: 97 active+clean; 195 MiB data, 810 MiB used, 714 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.6 MiB/s wr, 522 op/s 2024-09-21T12:26:27.764 INFO:teuthology.orchestra.run.smithi078.stdout: 5 26692 12.29 MB/sec warmup 65 sec latency 21.053 ms 2024-09-21T12:26:27.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:27 smithi148 ceph-mon[25818]: pgmap v273: 97 pgs: 97 active+clean; 195 MiB data, 810 MiB used, 714 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.6 MiB/s wr, 522 op/s 2024-09-21T12:26:28.764 INFO:teuthology.orchestra.run.smithi078.stdout: 5 27625 12.51 MB/sec warmup 66 sec latency 17.774 ms 2024-09-21T12:26:29.552 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:29 smithi078 ceph-mon[23681]: pgmap v274: 97 pgs: 97 active+clean; 206 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 5.9 KiB/s rd, 6.8 MiB/s wr, 658 op/s 2024-09-21T12:26:29.764 INFO:teuthology.orchestra.run.smithi078.stdout: 5 28490 12.83 MB/sec warmup 67 sec latency 15.495 ms 2024-09-21T12:26:29.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:29 smithi148 ceph-mon[25818]: pgmap v274: 97 pgs: 97 active+clean; 206 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 5.9 KiB/s rd, 6.8 MiB/s wr, 658 op/s 2024-09-21T12:26:30.764 INFO:teuthology.orchestra.run.smithi078.stdout: 5 29171 12.84 MB/sec warmup 68 sec latency 22.134 ms 2024-09-21T12:26:31.560 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:31 smithi078 ceph-mon[23681]: pgmap v275: 97 pgs: 97 active+clean; 218 MiB data, 860 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 850 op/s 2024-09-21T12:26:31.764 INFO:teuthology.orchestra.run.smithi078.stdout: 5 30050 12.91 MB/sec warmup 69 sec latency 17.310 ms 2024-09-21T12:26:31.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:31 smithi148 ceph-mon[25818]: pgmap v275: 97 pgs: 97 active+clean; 218 MiB data, 860 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 850 op/s 2024-09-21T12:26:32.764 INFO:teuthology.orchestra.run.smithi078.stdout: 5 30877 12.99 MB/sec warmup 70 sec latency 25.730 ms 2024-09-21T12:26:33.594 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:33 smithi078 ceph-mon[23681]: pgmap v276: 97 pgs: 97 active+clean; 218 MiB data, 898 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.6 MiB/s wr, 782 op/s 2024-09-21T12:26:33.764 INFO:teuthology.orchestra.run.smithi078.stdout: 5 31301 13.02 MB/sec warmup 71 sec latency 80.479 ms 2024-09-21T12:26:33.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:33 smithi148 ceph-mon[25818]: pgmap v276: 97 pgs: 97 active+clean; 218 MiB data, 898 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.6 MiB/s wr, 782 op/s 2024-09-21T12:26:34.764 INFO:teuthology.orchestra.run.smithi078.stdout: 5 31706 13.11 MB/sec warmup 72 sec latency 39.805 ms 2024-09-21T12:26:35.636 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:35 smithi078 ceph-mon[23681]: pgmap v277: 97 pgs: 97 active+clean; 217 MiB data, 903 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.4 MiB/s wr, 780 op/s 2024-09-21T12:26:35.765 INFO:teuthology.orchestra.run.smithi078.stdout: 5 32059 13.08 MB/sec warmup 73 sec latency 57.966 ms 2024-09-21T12:26:35.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:35 smithi148 ceph-mon[25818]: pgmap v277: 97 pgs: 97 active+clean; 217 MiB data, 903 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.4 MiB/s wr, 780 op/s 2024-09-21T12:26:36.764 INFO:teuthology.orchestra.run.smithi078.stdout: 5 32411 13.03 MB/sec warmup 74 sec latency 84.674 ms 2024-09-21T12:26:37.663 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:37 smithi078 ceph-mon[23681]: pgmap v278: 97 pgs: 97 active+clean; 219 MiB data, 971 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 998 op/s 2024-09-21T12:26:37.764 INFO:teuthology.orchestra.run.smithi078.stdout: 5 32719 12.92 MB/sec warmup 75 sec latency 64.432 ms 2024-09-21T12:26:37.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:37 smithi148 ceph-mon[25818]: pgmap v278: 97 pgs: 97 active+clean; 219 MiB data, 971 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 998 op/s 2024-09-21T12:26:38.764 INFO:teuthology.orchestra.run.smithi078.stdout: 5 33350 12.94 MB/sec warmup 76 sec latency 35.124 ms 2024-09-21T12:26:39.689 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:39 smithi078 ceph-mon[23681]: pgmap v279: 97 pgs: 97 active+clean; 226 MiB data, 1002 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 978 op/s 2024-09-21T12:26:39.765 INFO:teuthology.orchestra.run.smithi078.stdout: 5 33955 12.93 MB/sec warmup 77 sec latency 43.023 ms 2024-09-21T12:26:40.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:39 smithi148 ceph-mon[25818]: pgmap v279: 97 pgs: 97 active+clean; 226 MiB data, 1002 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 978 op/s 2024-09-21T12:26:40.764 INFO:teuthology.orchestra.run.smithi078.stdout: 5 34474 12.93 MB/sec warmup 78 sec latency 45.708 ms 2024-09-21T12:26:41.719 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:41 smithi078 ceph-mon[23681]: pgmap v280: 97 pgs: 97 active+clean; 232 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.8 MiB/s wr, 954 op/s 2024-09-21T12:26:41.764 INFO:teuthology.orchestra.run.smithi078.stdout: 5 34911 12.97 MB/sec warmup 79 sec latency 42.905 ms 2024-09-21T12:26:42.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:41 smithi148 ceph-mon[25818]: pgmap v280: 97 pgs: 97 active+clean; 232 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.8 MiB/s wr, 954 op/s 2024-09-21T12:26:42.764 INFO:teuthology.orchestra.run.smithi078.stdout: 5 35232 12.96 MB/sec warmup 80 sec latency 82.387 ms 2024-09-21T12:26:43.743 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:43 smithi078 ceph-mon[23681]: pgmap v281: 97 pgs: 97 active+clean; 228 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.1 MiB/s wr, 719 op/s 2024-09-21T12:26:43.765 INFO:teuthology.orchestra.run.smithi078.stdout: 5 35910 13.05 MB/sec warmup 81 sec latency 64.455 ms 2024-09-21T12:26:44.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:43 smithi148 ceph-mon[25818]: pgmap v281: 97 pgs: 97 active+clean; 228 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.1 MiB/s wr, 719 op/s 2024-09-21T12:26:44.765 INFO:teuthology.orchestra.run.smithi078.stdout: 5 36620 13.10 MB/sec warmup 82 sec latency 32.251 ms 2024-09-21T12:26:45.674 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:45 smithi078 ceph-mon[23681]: pgmap v282: 97 pgs: 97 active+clean; 226 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 650 op/s 2024-09-21T12:26:45.765 INFO:teuthology.orchestra.run.smithi078.stdout: 5 37450 13.12 MB/sec warmup 83 sec latency 24.984 ms 2024-09-21T12:26:45.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:45 smithi148 ceph-mon[25818]: pgmap v282: 97 pgs: 97 active+clean; 226 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 650 op/s 2024-09-21T12:26:46.765 INFO:teuthology.orchestra.run.smithi078.stdout: 5 38242 13.24 MB/sec warmup 84 sec latency 22.981 ms 2024-09-21T12:26:47.764 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:47 smithi078 ceph-mon[23681]: pgmap v283: 97 pgs: 97 active+clean; 240 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.7 MiB/s wr, 864 op/s 2024-09-21T12:26:47.765 INFO:teuthology.orchestra.run.smithi078.stdout: 5 39052 13.45 MB/sec warmup 85 sec latency 28.362 ms 2024-09-21T12:26:48.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:47 smithi148 ceph-mon[25818]: pgmap v283: 97 pgs: 97 active+clean; 240 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.7 MiB/s wr, 864 op/s 2024-09-21T12:26:48.765 INFO:teuthology.orchestra.run.smithi078.stdout: 5 39761 13.48 MB/sec warmup 86 sec latency 19.839 ms 2024-09-21T12:26:49.695 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:49 smithi148 ceph-mon[25818]: pgmap v284: 97 pgs: 97 active+clean; 245 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 8.0 MiB/s wr, 773 op/s 2024-09-21T12:26:49.765 INFO:teuthology.orchestra.run.smithi078.stdout: 5 40643 13.55 MB/sec warmup 87 sec latency 36.172 ms 2024-09-21T12:26:49.774 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:49 smithi078 ceph-mon[23681]: pgmap v284: 97 pgs: 97 active+clean; 245 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 8.0 MiB/s wr, 773 op/s 2024-09-21T12:26:50.765 INFO:teuthology.orchestra.run.smithi078.stdout: 5 41605 13.65 MB/sec warmup 88 sec latency 22.173 ms 2024-09-21T12:26:51.765 INFO:teuthology.orchestra.run.smithi078.stdout: 5 42386 13.81 MB/sec warmup 89 sec latency 20.207 ms 2024-09-21T12:26:51.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:51 smithi078 ceph-mon[23681]: pgmap v285: 97 pgs: 97 active+clean; 250 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.5 MiB/s wr, 869 op/s 2024-09-21T12:26:51.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:51 smithi078 ceph-mon[23681]: Upgrade: Updating mgr.smithi148.oyiqmx 2024-09-21T12:26:51.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:51 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:26:51.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:51 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi148.oyiqmx", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T12:26:51.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:51 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T12:26:51.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:51 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:26:51.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:51 smithi078 ceph-mon[23681]: Deploying daemon mgr.smithi148.oyiqmx on smithi148 2024-09-21T12:26:52.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:51 smithi148 ceph-mon[25818]: pgmap v285: 97 pgs: 97 active+clean; 250 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.5 MiB/s wr, 869 op/s 2024-09-21T12:26:52.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:51 smithi148 ceph-mon[25818]: Upgrade: Updating mgr.smithi148.oyiqmx 2024-09-21T12:26:52.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:51 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:26:52.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:51 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi148.oyiqmx", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T12:26:52.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:51 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T12:26:52.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:51 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:26:52.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:51 smithi148 ceph-mon[25818]: Deploying daemon mgr.smithi148.oyiqmx on smithi148 2024-09-21T12:26:52.765 INFO:teuthology.orchestra.run.smithi078.stdout: 5 42993 13.87 MB/sec warmup 90 sec latency 43.361 ms 2024-09-21T12:26:53.765 INFO:teuthology.orchestra.run.smithi078.stdout: 5 43429 13.81 MB/sec warmup 91 sec latency 43.325 ms 2024-09-21T12:26:53.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:53 smithi148 ceph-mon[25818]: pgmap v286: 97 pgs: 97 active+clean; 251 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.4 KiB/s rd, 8.7 MiB/s wr, 738 op/s 2024-09-21T12:26:53.803 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:53 smithi078 ceph-mon[23681]: pgmap v286: 97 pgs: 97 active+clean; 251 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.4 KiB/s rd, 8.7 MiB/s wr, 738 op/s 2024-09-21T12:26:54.313 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-21T12:26:54.765 INFO:teuthology.orchestra.run.smithi078.stdout: 5 43784 13.76 MB/sec warmup 92 sec latency 257.610 ms 2024-09-21T12:26:55.126 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T12:26:55.126 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (6m) 94s ago 8m 23.4M - ba2b418f427c 913ca01a7223 2024-09-21T12:26:55.126 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (8m) 94s ago 8m 6400k - 17.2.0 e1d6a67b021e 4a209e5da5ce 2024-09-21T12:26:55.127 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi148 smithi148 running (7m) 97s ago 7m 6405k - 17.2.0 e1d6a67b021e 0fa5e612307f 2024-09-21T12:26:55.127 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (6m) 94s ago 7m 51.1M - 8.3.5 dad864ee21e9 b6121d08c85b 2024-09-21T12:26:55.127 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.ytpzwh smithi078 running (2m) 94s ago 2m 14.8M - 17.2.0 e1d6a67b021e 5e9b109502a2 2024-09-21T12:26:55.127 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi148.uukezr smithi148 running (2m) 97s ago 2m 11.0M - 17.2.0 e1d6a67b021e b7e8aa0f3a6c 2024-09-21T12:26:55.127 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.ntwyan smithi078 *:9283 running (9m) 94s ago 9m 459M - 17.2.0 e1d6a67b021e caeba9628014 2024-09-21T12:26:55.127 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi148.oyiqmx smithi148 *:8443,9283 running (7m) 97s ago 7m 412M - 17.2.0 e1d6a67b021e 47b925ccff59 2024-09-21T12:26:55.127 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (9m) 94s ago 9m 46.5M 2048M 17.2.0 e1d6a67b021e 8261b0b2c7e0 2024-09-21T12:26:55.127 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi148 smithi148 running (7m) 97s ago 7m 37.7M 2048M 17.2.0 e1d6a67b021e fda961db230f 2024-09-21T12:26:55.127 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.ydbhru smithi078 *:2049 running (105s) 94s ago 105s 49.4M - 4.0 e1d6a67b021e 7e5b0274ac38 2024-09-21T12:26:55.127 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi148.einuzt smithi148 *:2049 running (100s) 97s ago 99s 50.7M - 4.0 e1d6a67b021e 76a8b0eabede 2024-09-21T12:26:55.127 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (7m) 94s ago 7m 16.5M - 1dbe0e931976 e7ca2ce65161 2024-09-21T12:26:55.127 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi148 smithi148 *:9100 running (7m) 97s ago 7m 17.4M - 1dbe0e931976 1474152580ab 2024-09-21T12:26:55.127 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (6m) 94s ago 6m 43.4M 3216M 17.2.0 e1d6a67b021e d4d5e9977070 2024-09-21T12:26:55.127 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (6m) 94s ago 6m 45.5M 3216M 17.2.0 e1d6a67b021e b9a4b952233b 2024-09-21T12:26:55.128 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (5m) 94s ago 5m 40.4M 3216M 17.2.0 e1d6a67b021e 829fb8c2d827 2024-09-21T12:26:55.128 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (5m) 94s ago 5m 40.7M 3216M 17.2.0 e1d6a67b021e ef942c8556d0 2024-09-21T12:26:55.128 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi148 running (4m) 97s ago 4m 41.0M 3984M 17.2.0 e1d6a67b021e 62eefdd572c0 2024-09-21T12:26:55.128 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi148 running (4m) 97s ago 4m 38.9M 3984M 17.2.0 e1d6a67b021e 7b44be78baca 2024-09-21T12:26:55.128 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi148 running (4m) 97s ago 4m 39.9M 3984M 17.2.0 e1d6a67b021e 3d23057910ea 2024-09-21T12:26:55.128 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi148 running (3m) 97s ago 3m 39.7M 3984M 17.2.0 e1d6a67b021e f9e4db9d35fa 2024-09-21T12:26:55.128 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (7m) 94s ago 7m 61.4M - 514e6a882f6e cdcd63cf2477 2024-09-21T12:26:55.616 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:26:55.616 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-21T12:26:55.617 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:26:55.617 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:26:55.617 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-21T12:26:55.617 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:26:55.617 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:26:55.617 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-21T12:26:55.617 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T12:26:55.617 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:26:55.617 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-21T12:26:55.617 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:26:55.617 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:26:55.617 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-21T12:26:55.617 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-21T12:26:55.617 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T12:26:55.618 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:26:55.765 INFO:teuthology.orchestra.run.smithi078.stdout: 5 44213 13.68 MB/sec warmup 93 sec latency 59.273 ms 2024-09-21T12:26:55.833 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:55 smithi078 ceph-mon[23681]: pgmap v287: 97 pgs: 97 active+clean; 255 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.4 KiB/s rd, 8.7 MiB/s wr, 779 op/s 2024-09-21T12:26:55.833 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:55 smithi078 ceph-mon[23681]: from='client.14666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:26:55.833 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:55 smithi078 ceph-mon[23681]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:26:56.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:55 smithi148 ceph-mon[25818]: pgmap v287: 97 pgs: 97 active+clean; 255 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.4 KiB/s rd, 8.7 MiB/s wr, 779 op/s 2024-09-21T12:26:56.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:55 smithi148 ceph-mon[25818]: from='client.14666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:26:56.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:55 smithi148 ceph-mon[25818]: from='client.14670 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:26:56.122 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:26:56.122 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5", 2024-09-21T12:26:56.123 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-21T12:26:56.123 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-09-21T12:26:56.123 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "0/23 daemons upgraded", 2024-09-21T12:26:56.123 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading mgr daemons" 2024-09-21T12:26:56.123 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:26:56.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:56 smithi078 ceph-mon[23681]: from='client.14674 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:26:56.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:56 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/990590094' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T12:26:56.672 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-21T12:26:56.765 INFO:teuthology.orchestra.run.smithi078.stdout: 5 44774 13.67 MB/sec warmup 94 sec latency 49.551 ms 2024-09-21T12:26:57.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:56 smithi148 ceph-mon[25818]: from='client.14674 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:26:57.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:56 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/990590094' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T12:26:57.612 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:57 smithi078 ceph-mon[23681]: pgmap v288: 97 pgs: 97 active+clean; 263 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 11 MiB/s wr, 1.05k op/s 2024-09-21T12:26:57.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:57 smithi078 ceph-mon[23681]: from='client.14682 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:26:57.613 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:57 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/1888302761' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T12:26:57.765 INFO:teuthology.orchestra.run.smithi078.stdout: 5 45309 13.69 MB/sec warmup 95 sec latency 53.321 ms 2024-09-21T12:26:57.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:57 smithi148 ceph-mon[25818]: pgmap v288: 97 pgs: 97 active+clean; 263 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 11 MiB/s wr, 1.05k op/s 2024-09-21T12:26:57.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:57 smithi148 ceph-mon[25818]: from='client.14682 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:26:57.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:57 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/1888302761' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T12:26:58.765 INFO:teuthology.orchestra.run.smithi078.stdout: 5 46007 13.87 MB/sec warmup 96 sec latency 37.585 ms 2024-09-21T12:26:59.622 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:26:59 smithi078 ceph-mon[23681]: pgmap v289: 97 pgs: 97 active+clean; 261 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 9.6 MiB/s wr, 908 op/s 2024-09-21T12:26:59.765 INFO:teuthology.orchestra.run.smithi078.stdout: 5 46744 13.91 MB/sec warmup 97 sec latency 23.605 ms 2024-09-21T12:26:59.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:26:59 smithi148 ceph-mon[25818]: pgmap v289: 97 pgs: 97 active+clean; 261 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 9.6 MiB/s wr, 908 op/s 2024-09-21T12:27:00.766 INFO:teuthology.orchestra.run.smithi078.stdout: 5 47301 13.89 MB/sec warmup 98 sec latency 44.057 ms 2024-09-21T12:27:01.620 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:01 smithi148 ceph-mon[25818]: pgmap v290: 97 pgs: 97 active+clean; 269 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.9 MiB/s wr, 891 op/s 2024-09-21T12:27:01.719 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:01 smithi078 ceph-mon[23681]: pgmap v290: 97 pgs: 97 active+clean; 269 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.9 MiB/s wr, 891 op/s 2024-09-21T12:27:01.765 INFO:teuthology.orchestra.run.smithi078.stdout: 5 47752 13.81 MB/sec warmup 99 sec latency 87.977 ms 2024-09-21T12:27:02.765 INFO:teuthology.orchestra.run.smithi078.stdout: 5 48141 13.76 MB/sec warmup 100 sec latency 61.041 ms 2024-09-21T12:27:03.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:03 smithi078 ceph-mon[23681]: pgmap v291: 97 pgs: 97 active+clean; 275 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 9.2 MiB/s wr, 768 op/s 2024-09-21T12:27:03.486 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:03 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:27:03.486 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:03 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:27:03.486 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:03 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:27:03.486 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:03 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:27:03.648 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:03 smithi148 ceph-mon[25818]: pgmap v291: 97 pgs: 97 active+clean; 275 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 9.2 MiB/s wr, 768 op/s 2024-09-21T12:27:03.648 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:03 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:27:03.648 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:03 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:27:03.648 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:03 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:27:03.648 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:03 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:27:03.765 INFO:teuthology.orchestra.run.smithi078.stdout: 5 48958 13.86 MB/sec warmup 101 sec latency 29.570 ms 2024-09-21T12:27:04.765 INFO:teuthology.orchestra.run.smithi078.stdout: 5 49524 13.96 MB/sec warmup 102 sec latency 32.868 ms 2024-09-21T12:27:05.498 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:05 smithi078 ceph-mon[23681]: pgmap v292: 97 pgs: 97 active+clean; 280 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.1 MiB/s wr, 790 op/s 2024-09-21T12:27:05.654 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:05 smithi148 ceph-mon[25818]: pgmap v292: 97 pgs: 97 active+clean; 280 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.1 MiB/s wr, 790 op/s 2024-09-21T12:27:05.766 INFO:teuthology.orchestra.run.smithi078.stdout: 5 50128 14.00 MB/sec warmup 103 sec latency 28.454 ms 2024-09-21T12:27:06.766 INFO:teuthology.orchestra.run.smithi078.stdout: 5 50633 13.96 MB/sec warmup 104 sec latency 31.055 ms 2024-09-21T12:27:07.466 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:07 smithi148 ceph-mon[25818]: pgmap v293: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.4 MiB/s wr, 868 op/s 2024-09-21T12:27:07.466 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:07 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:27:07.466 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:07 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:27:07.522 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:07 smithi078 ceph-mon[23681]: pgmap v293: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.4 MiB/s wr, 868 op/s 2024-09-21T12:27:07.522 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:07 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:27:07.522 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:07 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:27:07.770 INFO:teuthology.orchestra.run.smithi078.stdout: 5 51375 13.96 MB/sec warmup 105 sec latency 44.316 ms 2024-09-21T12:27:08.770 INFO:teuthology.orchestra.run.smithi078.stdout: 5 52159 14.03 MB/sec warmup 106 sec latency 22.684 ms 2024-09-21T12:27:09.537 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:09 smithi078 ceph-mon[23681]: pgmap v294: 97 pgs: 97 active+clean; 287 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 8.8 KiB/s rd, 8.1 MiB/s wr, 726 op/s 2024-09-21T12:27:09.537 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:09 smithi078 ceph-mon[23681]: Standby manager daemon smithi148.oyiqmx restarted 2024-09-21T12:27:09.537 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:09 smithi078 ceph-mon[23681]: Standby manager daemon smithi148.oyiqmx started 2024-09-21T12:27:09.537 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:09 smithi078 ceph-mon[23681]: from='mgr.? 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi148.oyiqmx/crt"}]: dispatch 2024-09-21T12:27:09.537 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:09 smithi078 ceph-mon[23681]: from='mgr.? 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T12:27:09.537 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:09 smithi078 ceph-mon[23681]: from='mgr.? 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi148.oyiqmx/key"}]: dispatch 2024-09-21T12:27:09.537 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:09 smithi078 ceph-mon[23681]: from='mgr.? 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T12:27:09.779 INFO:teuthology.orchestra.run.smithi078.stdout: 5 52838 14.12 MB/sec warmup 107 sec latency 37.820 ms 2024-09-21T12:27:09.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:09 smithi148 ceph-mon[25818]: pgmap v294: 97 pgs: 97 active+clean; 287 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 8.8 KiB/s rd, 8.1 MiB/s wr, 726 op/s 2024-09-21T12:27:09.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:09 smithi148 ceph-mon[25818]: Standby manager daemon smithi148.oyiqmx restarted 2024-09-21T12:27:09.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:09 smithi148 ceph-mon[25818]: Standby manager daemon smithi148.oyiqmx started 2024-09-21T12:27:09.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:09 smithi148 ceph-mon[25818]: from='mgr.? 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi148.oyiqmx/crt"}]: dispatch 2024-09-21T12:27:09.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:09 smithi148 ceph-mon[25818]: from='mgr.? 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T12:27:09.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:09 smithi148 ceph-mon[25818]: from='mgr.? 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi148.oyiqmx/key"}]: dispatch 2024-09-21T12:27:09.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:09 smithi148 ceph-mon[25818]: from='mgr.? 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T12:27:10.552 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:10 smithi078 ceph-mon[23681]: mgrmap e22: smithi078.ntwyan(active, since 7m), standbys: smithi148.oyiqmx 2024-09-21T12:27:10.764 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:10 smithi148 ceph-mon[25818]: mgrmap e22: smithi078.ntwyan(active, since 7m), standbys: smithi148.oyiqmx 2024-09-21T12:27:10.778 INFO:teuthology.orchestra.run.smithi078.stdout: 5 53295 14.16 MB/sec warmup 108 sec latency 33.040 ms 2024-09-21T12:27:11.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:11 smithi078 ceph-mon[23681]: pgmap v295: 97 pgs: 97 active+clean; 294 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.2 MiB/s wr, 857 op/s 2024-09-21T12:27:11.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:11 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:27:11.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:11 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:27:11.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:11 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:27:11.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:11 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:27:11.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:11 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mgr fail", "who": "smithi078.ntwyan"}]: dispatch 2024-09-21T12:27:11.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:11 smithi078 ceph-mon[23681]: osdmap e70: 8 total, 8 up, 8 in 2024-09-21T12:27:11.778 INFO:teuthology.orchestra.run.smithi078.stdout: 5 53865 14.14 MB/sec warmup 109 sec latency 22.690 ms 2024-09-21T12:27:11.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:11 smithi148 ceph-mon[25818]: pgmap v295: 97 pgs: 97 active+clean; 294 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.2 MiB/s wr, 857 op/s 2024-09-21T12:27:11.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:11 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:27:11.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:11 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:27:11.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:11 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' 2024-09-21T12:27:11.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:11 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:27:11.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:11 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mgr fail", "who": "smithi078.ntwyan"}]: dispatch 2024-09-21T12:27:11.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:11 smithi148 ceph-mon[25818]: osdmap e70: 8 total, 8 up, 8 in 2024-09-21T12:27:12.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:12 smithi148 ceph-mon[25818]: Upgrade: Need to upgrade myself (mgr.smithi078.ntwyan) 2024-09-21T12:27:12.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:12 smithi148 ceph-mon[25818]: Failing over to other MGR 2024-09-21T12:27:12.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:12 smithi148 ceph-mon[25818]: Standby manager daemon smithi078.ntwyan started 2024-09-21T12:27:12.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:12 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-21T12:27:12.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:12 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "mon metadata", "id": "smithi148"}]: dispatch 2024-09-21T12:27:12.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:12 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.ytpzwh"}]: dispatch 2024-09-21T12:27:12.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:12 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi148.uukezr"}]: dispatch 2024-09-21T12:27:12.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:12 smithi148 ceph-mon[25818]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "mgr fail", "who": "smithi078.ntwyan"}]': finished 2024-09-21T12:27:12.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:12 smithi148 ceph-mon[25818]: mgrmap e23: smithi148.oyiqmx(active, starting, since 0.999457s), standbys: smithi078.ntwyan 2024-09-21T12:27:12.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:12 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "mgr metadata", "who": "smithi148.oyiqmx", "id": "smithi148.oyiqmx"}]: dispatch 2024-09-21T12:27:12.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:12 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "mgr metadata", "who": "smithi078.ntwyan", "id": "smithi078.ntwyan"}]: dispatch 2024-09-21T12:27:12.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:12 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T12:27:12.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:12 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T12:27:12.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:12 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T12:27:12.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:12 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T12:27:12.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:12 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T12:27:12.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:12 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T12:27:12.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:12 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T12:27:12.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:12 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T12:27:12.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:12 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T12:27:12.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:12 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T12:27:12.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:12 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T12:27:12.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:12 smithi148 ceph-mon[25818]: Manager daemon smithi148.oyiqmx is now available 2024-09-21T12:27:12.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:12 smithi078 ceph-mon[23681]: Upgrade: Need to upgrade myself (mgr.smithi078.ntwyan) 2024-09-21T12:27:12.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:12 smithi078 ceph-mon[23681]: Failing over to other MGR 2024-09-21T12:27:12.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:12 smithi078 ceph-mon[23681]: Standby manager daemon smithi078.ntwyan started 2024-09-21T12:27:12.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:12 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-21T12:27:12.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:12 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "mon metadata", "id": "smithi148"}]: dispatch 2024-09-21T12:27:12.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:12 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.ytpzwh"}]: dispatch 2024-09-21T12:27:12.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:12 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi148.uukezr"}]: dispatch 2024-09-21T12:27:12.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:12 smithi078 ceph-mon[23681]: from='mgr.14202 172.21.15.78:0/3863191591' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "mgr fail", "who": "smithi078.ntwyan"}]': finished 2024-09-21T12:27:12.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:12 smithi078 ceph-mon[23681]: mgrmap e23: smithi148.oyiqmx(active, starting, since 0.999457s), standbys: smithi078.ntwyan 2024-09-21T12:27:12.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:12 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "mgr metadata", "who": "smithi148.oyiqmx", "id": "smithi148.oyiqmx"}]: dispatch 2024-09-21T12:27:12.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:12 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "mgr metadata", "who": "smithi078.ntwyan", "id": "smithi078.ntwyan"}]: dispatch 2024-09-21T12:27:12.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:12 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T12:27:12.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:12 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T12:27:12.573 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:12 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T12:27:12.573 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:12 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T12:27:12.573 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:12 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T12:27:12.573 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:12 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T12:27:12.573 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:12 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T12:27:12.573 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:12 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T12:27:12.573 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:12 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T12:27:12.573 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:12 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T12:27:12.573 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:12 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T12:27:12.573 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:12 smithi078 ceph-mon[23681]: Manager daemon smithi148.oyiqmx is now available 2024-09-21T12:27:12.779 INFO:teuthology.orchestra.run.smithi078.stdout: 5 54546 14.13 MB/sec warmup 110 sec latency 22.205 ms 2024-09-21T12:27:13.779 INFO:teuthology.orchestra.run.smithi078.stdout: 5 55252 14.12 MB/sec warmup 111 sec latency 32.020 ms 2024-09-21T12:27:14.310 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:14 smithi078 ceph-mon[23681]: mgrmap e24: smithi148.oyiqmx(active, since 2s), standbys: smithi078.ntwyan 2024-09-21T12:27:14.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:14 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:14.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:14 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:14.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:14 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:14.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:14 smithi078 ceph-mon[23681]: Checking for cert/key for grafana.smithi078 2024-09-21T12:27:14.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:14 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:14.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:14 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:27:14.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:14 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T12:27:14.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:14 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T12:27:14.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:14 smithi078 ceph-mon[23681]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T12:27:14.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:14 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi148.oyiqmx/mirror_snapshot_schedule"}]: dispatch 2024-09-21T12:27:14.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:14 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi148.oyiqmx/mirror_snapshot_schedule"}]: dispatch 2024-09-21T12:27:14.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:14 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi148.oyiqmx/trash_purge_schedule"}]: dispatch 2024-09-21T12:27:14.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:14 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi148.oyiqmx/trash_purge_schedule"}]: dispatch 2024-09-21T12:27:14.312 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:14 smithi078 ceph-mon[23681]: Deploying cephadm binary to smithi148 2024-09-21T12:27:14.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:14 smithi148 ceph-mon[25818]: mgrmap e24: smithi148.oyiqmx(active, since 2s), standbys: smithi078.ntwyan 2024-09-21T12:27:14.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:14 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:14.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:14 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:14.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:14 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:14.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:14 smithi148 ceph-mon[25818]: Checking for cert/key for grafana.smithi078 2024-09-21T12:27:14.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:14 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:14.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:14 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:27:14.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:14 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T12:27:14.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:14 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T12:27:14.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:14 smithi148 ceph-mon[25818]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T12:27:14.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:14 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi148.oyiqmx/mirror_snapshot_schedule"}]: dispatch 2024-09-21T12:27:14.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:14 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi148.oyiqmx/mirror_snapshot_schedule"}]: dispatch 2024-09-21T12:27:14.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:14 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi148.oyiqmx/trash_purge_schedule"}]: dispatch 2024-09-21T12:27:14.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:14 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi148.oyiqmx/trash_purge_schedule"}]: dispatch 2024-09-21T12:27:14.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:14 smithi148 ceph-mon[25818]: Deploying cephadm binary to smithi148 2024-09-21T12:27:14.779 INFO:teuthology.orchestra.run.smithi078.stdout: 5 55950 14.17 MB/sec warmup 112 sec latency 29.254 ms 2024-09-21T12:27:15.318 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:15 smithi078 ceph-mon[23681]: pgmap v4: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T12:27:15.318 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:15 smithi078 ceph-mon[23681]: [21/Sep/2024:12:27:14] ENGINE Bus STARTING 2024-09-21T12:27:15.318 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:15 smithi078 ceph-mon[23681]: [21/Sep/2024:12:27:14] ENGINE Serving on http://172.21.15.148:8765 2024-09-21T12:27:15.318 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:15 smithi078 ceph-mon[23681]: [21/Sep/2024:12:27:14] ENGINE Serving on https://172.21.15.148:7150 2024-09-21T12:27:15.319 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:15 smithi078 ceph-mon[23681]: [21/Sep/2024:12:27:14] ENGINE Bus STARTED 2024-09-21T12:27:15.319 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:15 smithi078 ceph-mon[23681]: [21/Sep/2024:12:27:14] ENGINE Error in HTTPServer.serve 2024-09-21T12:27:15.319 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Traceback (most recent call last): 2024-09-21T12:27:15.319 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T12:27:15.319 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self._connections.run(self.expiration_interval) 2024-09-21T12:27:15.319 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T12:27:15.319 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self._run(expiration_interval) 2024-09-21T12:27:15.319 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T12:27:15.319 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T12:27:15.319 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T12:27:15.319 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T12:27:15.319 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T12:27:15.319 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: s = self.context.wrap_socket( 2024-09-21T12:27:15.319 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T12:27:15.320 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: return self.sslsocket_class._create( 2024-09-21T12:27:15.320 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T12:27:15.320 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self.do_handshake() 2024-09-21T12:27:15.320 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T12:27:15.320 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self._sslobj.do_handshake() 2024-09-21T12:27:15.320 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T12:27:15.320 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:15 smithi078 ceph-mon[23681]: Deploying cephadm binary to smithi078 2024-09-21T12:27:15.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:15 smithi148 ceph-mon[25818]: pgmap v4: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T12:27:15.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:15 smithi148 ceph-mon[25818]: [21/Sep/2024:12:27:14] ENGINE Bus STARTING 2024-09-21T12:27:15.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:15 smithi148 ceph-mon[25818]: [21/Sep/2024:12:27:14] ENGINE Serving on http://172.21.15.148:8765 2024-09-21T12:27:15.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:15 smithi148 ceph-mon[25818]: [21/Sep/2024:12:27:14] ENGINE Serving on https://172.21.15.148:7150 2024-09-21T12:27:15.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:15 smithi148 ceph-mon[25818]: [21/Sep/2024:12:27:14] ENGINE Bus STARTED 2024-09-21T12:27:15.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:15 smithi148 ceph-mon[25818]: [21/Sep/2024:12:27:14] ENGINE Error in HTTPServer.serve 2024-09-21T12:27:15.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Traceback (most recent call last): 2024-09-21T12:27:15.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T12:27:15.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: self._connections.run(self.expiration_interval) 2024-09-21T12:27:15.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T12:27:15.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: self._run(expiration_interval) 2024-09-21T12:27:15.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T12:27:15.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T12:27:15.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T12:27:15.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T12:27:15.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T12:27:15.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: s = self.context.wrap_socket( 2024-09-21T12:27:15.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T12:27:15.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: return self.sslsocket_class._create( 2024-09-21T12:27:15.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T12:27:15.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: self.do_handshake() 2024-09-21T12:27:15.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T12:27:15.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: self._sslobj.do_handshake() 2024-09-21T12:27:15.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T12:27:15.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:15 smithi148 ceph-mon[25818]: Deploying cephadm binary to smithi078 2024-09-21T12:27:15.779 INFO:teuthology.orchestra.run.smithi078.stdout: 5 56426 14.24 MB/sec warmup 113 sec latency 43.127 ms 2024-09-21T12:27:16.329 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:16 smithi078 ceph-mon[23681]: mgrmap e25: smithi148.oyiqmx(active, since 4s), standbys: smithi078.ntwyan 2024-09-21T12:27:16.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:16 smithi148 ceph-mon[25818]: mgrmap e25: smithi148.oyiqmx(active, since 4s), standbys: smithi078.ntwyan 2024-09-21T12:27:16.779 INFO:teuthology.orchestra.run.smithi078.stdout: 5 56851 14.25 MB/sec warmup 114 sec latency 71.779 ms 2024-09-21T12:27:17.779 INFO:teuthology.orchestra.run.smithi078.stdout: 5 57322 14.21 MB/sec warmup 115 sec latency 41.503 ms 2024-09-21T12:27:18.349 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:18 smithi078 ceph-mon[23681]: pgmap v5: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T12:27:18.349 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:18 smithi078 ceph-mon[23681]: mgrmap e26: smithi148.oyiqmx(active, since 6s), standbys: smithi078.ntwyan 2024-09-21T12:27:18.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:18 smithi148 ceph-mon[25818]: pgmap v5: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T12:27:18.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:18 smithi148 ceph-mon[25818]: mgrmap e26: smithi148.oyiqmx(active, since 6s), standbys: smithi078.ntwyan 2024-09-21T12:27:18.779 INFO:teuthology.orchestra.run.smithi078.stdout: 5 57819 14.19 MB/sec warmup 116 sec latency 29.029 ms 2024-09-21T12:27:19.779 INFO:teuthology.orchestra.run.smithi078.stdout: 5 58309 14.11 MB/sec warmup 117 sec latency 48.291 ms 2024-09-21T12:27:20.368 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:20 smithi078 ceph-mon[23681]: pgmap v6: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 12 KiB/s wr, 4 op/s 2024-09-21T12:27:20.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:20 smithi148 ceph-mon[25818]: pgmap v6: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 12 KiB/s wr, 4 op/s 2024-09-21T12:27:20.779 INFO:teuthology.orchestra.run.smithi078.stdout: 5 58692 14.08 MB/sec warmup 118 sec latency 60.404 ms 2024-09-21T12:27:21.779 INFO:teuthology.orchestra.run.smithi078.stdout: 5 59180 14.08 MB/sec warmup 119 sec latency 55.508 ms 2024-09-21T12:27:22.409 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:22 smithi078 ceph-mon[23681]: pgmap v7: 97 pgs: 97 active+clean; 315 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 6.0 MiB/s wr, 530 op/s 2024-09-21T12:27:22.409 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:22 smithi078 ceph-mon[23681]: Standby manager daemon smithi078.ntwyan restarted 2024-09-21T12:27:22.409 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:22 smithi078 ceph-mon[23681]: Standby manager daemon smithi078.ntwyan started 2024-09-21T12:27:22.409 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:22 smithi078 ceph-mon[23681]: from='mgr.? 172.21.15.78:0/3442159397' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.ntwyan/crt"}]: dispatch 2024-09-21T12:27:22.409 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:22 smithi078 ceph-mon[23681]: from='mgr.? 172.21.15.78:0/3442159397' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T12:27:22.409 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:22 smithi078 ceph-mon[23681]: from='mgr.? 172.21.15.78:0/3442159397' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.ntwyan/key"}]: dispatch 2024-09-21T12:27:22.409 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:22 smithi078 ceph-mon[23681]: from='mgr.? 172.21.15.78:0/3442159397' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T12:27:22.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:22 smithi148 ceph-mon[25818]: pgmap v7: 97 pgs: 97 active+clean; 315 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 6.0 MiB/s wr, 530 op/s 2024-09-21T12:27:22.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:22 smithi148 ceph-mon[25818]: Standby manager daemon smithi078.ntwyan restarted 2024-09-21T12:27:22.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:22 smithi148 ceph-mon[25818]: Standby manager daemon smithi078.ntwyan started 2024-09-21T12:27:22.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:22 smithi148 ceph-mon[25818]: from='mgr.? 172.21.15.78:0/3442159397' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.ntwyan/crt"}]: dispatch 2024-09-21T12:27:22.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:22 smithi148 ceph-mon[25818]: from='mgr.? 172.21.15.78:0/3442159397' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T12:27:22.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:22 smithi148 ceph-mon[25818]: from='mgr.? 172.21.15.78:0/3442159397' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.ntwyan/key"}]: dispatch 2024-09-21T12:27:22.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:22 smithi148 ceph-mon[25818]: from='mgr.? 172.21.15.78:0/3442159397' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T12:27:23.425 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:23 smithi078 ceph-mon[23681]: mgrmap e27: smithi148.oyiqmx(active, since 10s), standbys: smithi078.ntwyan 2024-09-21T12:27:23.426 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:23 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:23.426 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:23 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:23.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:23 smithi148 ceph-mon[25818]: mgrmap e27: smithi148.oyiqmx(active, since 10s), standbys: smithi078.ntwyan 2024-09-21T12:27:23.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:23 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:23.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:23 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:23.779 INFO:teuthology.orchestra.run.smithi078.stdout: 5 60251 23.84 MB/sec execute 1 sec latency 53.946 ms 2024-09-21T12:27:24.181 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:24 smithi078 ceph-mon[23681]: pgmap v8: 97 pgs: 97 active+clean; 315 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.7 MiB/s wr, 412 op/s 2024-09-21T12:27:24.181 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:24 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:24.181 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:24 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:24.181 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:24 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:24.181 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:24 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:24.181 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:24 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:24.181 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:24 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:24.181 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:24 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:24.181 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:24 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:24.182 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:24 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:24.182 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:24 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:24.182 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:24 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:24.182 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:24 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:24.182 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:24 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:24.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:24 smithi148 ceph-mon[25818]: pgmap v8: 97 pgs: 97 active+clean; 315 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.7 MiB/s wr, 412 op/s 2024-09-21T12:27:24.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:24 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:24.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:24 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:24.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:24 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:24.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:24 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:24.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:24 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:24.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:24 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:24.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:24 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:24.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:24 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:24.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:24 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:24.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:24 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:24.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:24 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:24.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:24 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:24.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:24 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:24.779 INFO:teuthology.orchestra.run.smithi078.stdout: 5 60724 19.40 MB/sec execute 2 sec latency 32.435 ms 2024-09-21T12:27:25.214 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:25 smithi078 ceph-mon[23681]: Adjusting osd_memory_target on smithi078 to 1936M 2024-09-21T12:27:25.214 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:25 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:25.214 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:25 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:25.457 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:25 smithi148 ceph-mon[25818]: Adjusting osd_memory_target on smithi078 to 1936M 2024-09-21T12:27:25.457 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:25 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:25.457 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:25 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:25.779 INFO:teuthology.orchestra.run.smithi078.stdout: 5 60968 14.25 MB/sec execute 3 sec latency 92.524 ms 2024-09-21T12:27:26.225 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:26 smithi078 ceph-mon[23681]: pgmap v9: 97 pgs: 97 active+clean; 319 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 6.9 MiB/s wr, 652 op/s 2024-09-21T12:27:26.518 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:26 smithi148 ceph-mon[25818]: pgmap v9: 97 pgs: 97 active+clean; 319 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 6.9 MiB/s wr, 652 op/s 2024-09-21T12:27:26.779 INFO:teuthology.orchestra.run.smithi078.stdout: 5 61395 13.02 MB/sec execute 4 sec latency 43.322 ms 2024-09-21T12:27:27.144 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-21T12:27:27.779 INFO:teuthology.orchestra.run.smithi078.stdout: 5 61846 11.40 MB/sec execute 5 sec latency 93.284 ms 2024-09-21T12:27:28.159 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T12:27:28.160 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (7m) 5s ago 8m 29.0M - 0.23.0 ba2b418f427c 913ca01a7223 2024-09-21T12:27:28.160 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (8m) 5s ago 8m 6400k - 17.2.0 e1d6a67b021e 4a209e5da5ce 2024-09-21T12:27:28.160 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi148 smithi148 running (8m) 4s ago 8m 6405k - 17.2.0 e1d6a67b021e 0fa5e612307f 2024-09-21T12:27:28.160 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (7m) 5s ago 8m 52.6M - dad864ee21e9 b6121d08c85b 2024-09-21T12:27:28.160 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.ytpzwh smithi078 running (2m) 5s ago 2m 34.5M - 17.2.0 e1d6a67b021e 5e9b109502a2 2024-09-21T12:27:28.160 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi148.uukezr smithi148 running (2m) 4s ago 2m 12.0M - 17.2.0 e1d6a67b021e b7e8aa0f3a6c 2024-09-21T12:27:28.160 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.ntwyan smithi078 *:9283 running (9m) 5s ago 9m 195M - 17.2.0 e1d6a67b021e caeba9628014 2024-09-21T12:27:28.160 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi148.oyiqmx smithi148 *:8443,9283 running (26s) 4s ago 8m 496M - 19.3.0-5104-g44178494 b415816db8b4 58467ce9a26b 2024-09-21T12:27:28.160 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (9m) 5s ago 10m 52.3M 2048M 17.2.0 e1d6a67b021e 8261b0b2c7e0 2024-09-21T12:27:28.160 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi148 smithi148 running (7m) 4s ago 7m 42.0M 2048M 17.2.0 e1d6a67b021e fda961db230f 2024-09-21T12:27:28.160 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.ydbhru smithi078 *:2049 running (2m) 5s ago 2m 239M - 4.0 e1d6a67b021e 7e5b0274ac38 2024-09-21T12:27:28.161 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi148.einuzt smithi148 *:2049 running (2m) 4s ago 2m 76.0M - 4.0 e1d6a67b021e 76a8b0eabede 2024-09-21T12:27:28.161 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (8m) 5s ago 8m 17.5M - 1.3.1 1dbe0e931976 e7ca2ce65161 2024-09-21T12:27:28.161 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi148 smithi148 *:9100 running (7m) 4s ago 7m 21.8M - 1.3.1 1dbe0e931976 1474152580ab 2024-09-21T12:27:28.161 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (7m) 5s ago 7m 265M 1936M 17.2.0 e1d6a67b021e d4d5e9977070 2024-09-21T12:27:28.161 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (6m) 5s ago 6m 196M 1936M 17.2.0 e1d6a67b021e b9a4b952233b 2024-09-21T12:27:28.161 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (6m) 5s ago 6m 171M 1936M 17.2.0 e1d6a67b021e 829fb8c2d827 2024-09-21T12:27:28.161 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (5m) 5s ago 5m 214M 1936M 17.2.0 e1d6a67b021e ef942c8556d0 2024-09-21T12:27:28.161 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi148 running (5m) 4s ago 5m 230M 3984M 17.2.0 e1d6a67b021e 62eefdd572c0 2024-09-21T12:27:28.161 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi148 running (4m) 4s ago 4m 234M 3984M 17.2.0 e1d6a67b021e 7b44be78baca 2024-09-21T12:27:28.161 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi148 running (4m) 4s ago 4m 224M 3984M 17.2.0 e1d6a67b021e 3d23057910ea 2024-09-21T12:27:28.161 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi148 running (4m) 4s ago 4m 314M 3984M 17.2.0 e1d6a67b021e f9e4db9d35fa 2024-09-21T12:27:28.161 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (7m) 5s ago 7m 63.8M - 2.33.4 514e6a882f6e cdcd63cf2477 2024-09-21T12:27:28.262 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:28 smithi078 ceph-mon[23681]: pgmap v10: 97 pgs: 97 active+clean; 319 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 6.3 MiB/s wr, 598 op/s 2024-09-21T12:27:28.262 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:28 smithi078 ceph-mon[23681]: from='client.14716 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:27:28.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:28 smithi148 ceph-mon[25818]: pgmap v10: 97 pgs: 97 active+clean; 319 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 6.3 MiB/s wr, 598 op/s 2024-09-21T12:27:28.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:28 smithi148 ceph-mon[25818]: from='client.14716 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:27:28.718 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:27:28.719 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-21T12:27:28.719 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:27:28.719 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:27:28.719 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-21T12:27:28.719 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-21T12:27:28.719 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 1 2024-09-21T12:27:28.719 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:27:28.719 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-21T12:27:28.719 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T12:27:28.719 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:27:28.719 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-21T12:27:28.719 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:27:28.719 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:27:28.719 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-21T12:27:28.720 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-21T12:27:28.720 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 1 2024-09-21T12:27:28.720 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T12:27:28.720 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:27:28.779 INFO:teuthology.orchestra.run.smithi078.stdout: 5 62225 11.24 MB/sec execute 6 sec latency 57.330 ms 2024-09-21T12:27:29.171 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:27:29.171 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5", 2024-09-21T12:27:29.171 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-21T12:27:29.171 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T12:27:29.171 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-09-21T12:27:29.171 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "1/23 daemons upgraded", 2024-09-21T12:27:29.171 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "", 2024-09-21T12:27:29.171 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-21T12:27:29.171 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:27:29.291 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:29 smithi078 ceph-mon[23681]: from='client.14720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:27:29.291 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:29 smithi078 ceph-mon[23681]: from='client.14724 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:27:29.291 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:29 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T12:27:29.291 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:29 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/2362516572' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T12:27:29.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:29 smithi148 ceph-mon[25818]: from='client.14720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:27:29.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:29 smithi148 ceph-mon[25818]: from='client.14724 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:27:29.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:29 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T12:27:29.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:29 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/2362516572' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T12:27:29.779 INFO:teuthology.orchestra.run.smithi078.stdout: 5 62719 11.55 MB/sec execute 7 sec latency 40.848 ms 2024-09-21T12:27:29.831 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-21T12:27:30.310 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:30 smithi078 ceph-mon[23681]: pgmap v11: 97 pgs: 97 active+clean; 321 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 7.6 MiB/s wr, 747 op/s 2024-09-21T12:27:30.310 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:30 smithi078 ceph-mon[23681]: from='client.14732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:27:30.310 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:30 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:30.310 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:30 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:30.310 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:30 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:30.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:30 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:30.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:30 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:30.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:30 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:30.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:30 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:30.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:30 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:30.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:30 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:30.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:30 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:30.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:30 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:30.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:30 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:27:30.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:30 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:27:30.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:30 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/3107707373' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T12:27:30.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:30 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:30.311 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:30 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:30.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:30 smithi148 ceph-mon[25818]: pgmap v11: 97 pgs: 97 active+clean; 321 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 7.6 MiB/s wr, 747 op/s 2024-09-21T12:27:30.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:30 smithi148 ceph-mon[25818]: from='client.14732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:27:30.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:30 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:30.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:30 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:30.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:30 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:30.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:30 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:30.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:30 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:30.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:30 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:30.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:30 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:30.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:30 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:30.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:30 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:30.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:30 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T12:27:30.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:30 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:30.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:30 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:27:30.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:30 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:27:30.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:30 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/3107707373' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T12:27:30.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:30 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:30.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:30 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:30.780 INFO:teuthology.orchestra.run.smithi078.stdout: 5 63249 12.03 MB/sec execute 8 sec latency 59.590 ms 2024-09-21T12:27:31.321 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:31 smithi078 ceph-mon[23681]: Adjusting osd_memory_target on smithi148 to 2704M 2024-09-21T12:27:31.321 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:31 smithi078 ceph-mon[23681]: Updating smithi078:/etc/ceph/ceph.conf 2024-09-21T12:27:31.321 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:31 smithi078 ceph-mon[23681]: Updating smithi148:/etc/ceph/ceph.conf 2024-09-21T12:27:31.321 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:31 smithi078 ceph-mon[23681]: Updating smithi148:/var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T12:27:31.321 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:31 smithi078 ceph-mon[23681]: Updating smithi078:/var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T12:27:31.321 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:31 smithi078 ceph-mon[23681]: Updating smithi148:/etc/ceph/ceph.client.admin.keyring 2024-09-21T12:27:31.321 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:31 smithi078 ceph-mon[23681]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-09-21T12:27:31.322 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:31 smithi078 ceph-mon[23681]: Updating smithi148:/var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T12:27:31.322 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:31 smithi078 ceph-mon[23681]: Updating smithi078:/var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T12:27:31.322 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:31 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:31.322 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:31 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:31.322 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:31 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:31.322 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:31 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:31.322 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:31 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:31.322 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:31 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:31.322 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:31 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:31.322 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:31 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:31.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:31 smithi148 ceph-mon[25818]: Adjusting osd_memory_target on smithi148 to 2704M 2024-09-21T12:27:31.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:31 smithi148 ceph-mon[25818]: Updating smithi078:/etc/ceph/ceph.conf 2024-09-21T12:27:31.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:31 smithi148 ceph-mon[25818]: Updating smithi148:/etc/ceph/ceph.conf 2024-09-21T12:27:31.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:31 smithi148 ceph-mon[25818]: Updating smithi148:/var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T12:27:31.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:31 smithi148 ceph-mon[25818]: Updating smithi078:/var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T12:27:31.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:31 smithi148 ceph-mon[25818]: Updating smithi148:/etc/ceph/ceph.client.admin.keyring 2024-09-21T12:27:31.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:31 smithi148 ceph-mon[25818]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-09-21T12:27:31.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:31 smithi148 ceph-mon[25818]: Updating smithi148:/var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T12:27:31.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:31 smithi148 ceph-mon[25818]: Updating smithi078:/var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T12:27:31.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:31 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:31.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:31 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:31.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:31 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:31.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:31 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:31.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:31 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:31.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:31 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:31.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:31 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:31.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:31 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:31.780 INFO:teuthology.orchestra.run.smithi078.stdout: 5 63914 13.73 MB/sec execute 9 sec latency 19.919 ms 2024-09-21T12:27:32.325 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:32 smithi078 ceph-mon[23681]: pgmap v12: 97 pgs: 97 active+clean; 324 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 8.8 MiB/s wr, 917 op/s 2024-09-21T12:27:32.325 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:32 smithi078 ceph-mon[23681]: Reconfiguring alertmanager.smithi078 (dependencies changed)... 2024-09-21T12:27:32.325 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:32 smithi078 ceph-mon[23681]: Deploying daemon alertmanager.smithi078 on smithi078 2024-09-21T12:27:32.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:32 smithi148 ceph-mon[25818]: pgmap v12: 97 pgs: 97 active+clean; 324 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 8.8 MiB/s wr, 917 op/s 2024-09-21T12:27:32.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:32 smithi148 ceph-mon[25818]: Reconfiguring alertmanager.smithi078 (dependencies changed)... 2024-09-21T12:27:32.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:32 smithi148 ceph-mon[25818]: Deploying daemon alertmanager.smithi078 on smithi078 2024-09-21T12:27:32.780 INFO:teuthology.orchestra.run.smithi078.stdout: 5 64419 13.68 MB/sec execute 10 sec latency 22.858 ms 2024-09-21T12:27:33.780 INFO:teuthology.orchestra.run.smithi078.stdout: 5 64946 13.46 MB/sec execute 11 sec latency 26.834 ms 2024-09-21T12:27:34.340 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:34 smithi078 ceph-mon[23681]: pgmap v13: 97 pgs: 97 active+clean; 324 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.3 MiB/s wr, 610 op/s 2024-09-21T12:27:34.424 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:34 smithi148 ceph-mon[25818]: pgmap v13: 97 pgs: 97 active+clean; 324 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.3 MiB/s wr, 610 op/s 2024-09-21T12:27:34.780 INFO:teuthology.orchestra.run.smithi078.stdout: 5 65474 13.06 MB/sec execute 12 sec latency 36.815 ms 2024-09-21T12:27:35.780 INFO:teuthology.orchestra.run.smithi078.stdout: 5 65993 12.88 MB/sec execute 13 sec latency 39.108 ms 2024-09-21T12:27:36.233 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:36 smithi078 ceph-mon[23681]: pgmap v14: 97 pgs: 97 active+clean; 337 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 917 op/s 2024-09-21T12:27:36.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:36 smithi148 ceph-mon[25818]: pgmap v14: 97 pgs: 97 active+clean; 337 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.8 MiB/s wr, 917 op/s 2024-09-21T12:27:36.780 INFO:teuthology.orchestra.run.smithi078.stdout: 5 66469 12.99 MB/sec execute 14 sec latency 36.718 ms 2024-09-21T12:27:37.780 INFO:teuthology.orchestra.run.smithi078.stdout: 5 66953 13.47 MB/sec execute 15 sec latency 105.735 ms 2024-09-21T12:27:38.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:38 smithi078 ceph-mon[23681]: pgmap v15: 97 pgs: 97 active+clean; 337 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.0 MiB/s wr, 628 op/s 2024-09-21T12:27:38.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:38 smithi148 ceph-mon[25818]: pgmap v15: 97 pgs: 97 active+clean; 337 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.0 MiB/s wr, 628 op/s 2024-09-21T12:27:38.780 INFO:teuthology.orchestra.run.smithi078.stdout: 5 67420 13.66 MB/sec execute 16 sec latency 64.035 ms 2024-09-21T12:27:39.780 INFO:teuthology.orchestra.run.smithi078.stdout: 5 67937 13.66 MB/sec execute 17 sec latency 29.961 ms 2024-09-21T12:27:40.411 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:40 smithi078 ceph-mon[23681]: pgmap v16: 97 pgs: 97 active+clean; 321 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 7.1 MiB/s wr, 687 op/s 2024-09-21T12:27:40.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:40 smithi148 ceph-mon[25818]: pgmap v16: 97 pgs: 97 active+clean; 321 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 7.1 MiB/s wr, 687 op/s 2024-09-21T12:27:40.780 INFO:teuthology.orchestra.run.smithi078.stdout: 5 68321 13.17 MB/sec execute 18 sec latency 33.543 ms 2024-09-21T12:27:41.426 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:41 smithi078 ceph-mon[23681]: pgmap v17: 97 pgs: 97 active+clean; 320 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.4 MiB/s wr, 761 op/s 2024-09-21T12:27:41.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:41 smithi148 ceph-mon[25818]: pgmap v17: 97 pgs: 97 active+clean; 320 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.4 MiB/s wr, 761 op/s 2024-09-21T12:27:41.780 INFO:teuthology.orchestra.run.smithi078.stdout: 5 68912 13.20 MB/sec execute 19 sec latency 37.655 ms 2024-09-21T12:27:42.780 INFO:teuthology.orchestra.run.smithi078.stdout: 5 69538 13.13 MB/sec execute 20 sec latency 54.600 ms 2024-09-21T12:27:43.780 INFO:teuthology.orchestra.run.smithi078.stdout: 5 70126 13.46 MB/sec execute 21 sec latency 87.000 ms 2024-09-21T12:27:44.208 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:44 smithi078 ceph-mon[23681]: pgmap v18: 97 pgs: 97 active+clean; 320 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.2 MiB/s wr, 589 op/s 2024-09-21T12:27:44.209 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:44 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T12:27:44.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:44 smithi148 ceph-mon[25818]: pgmap v18: 97 pgs: 97 active+clean; 320 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.2 MiB/s wr, 589 op/s 2024-09-21T12:27:44.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:44 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T12:27:44.780 INFO:teuthology.orchestra.run.smithi078.stdout: 5 70774 13.93 MB/sec execute 22 sec latency 31.505 ms 2024-09-21T12:27:45.239 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:45 smithi078 ceph-mon[23681]: pgmap v19: 97 pgs: 97 active+clean; 315 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.3 MiB/s wr, 920 op/s 2024-09-21T12:27:45.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:45 smithi148 ceph-mon[25818]: pgmap v19: 97 pgs: 97 active+clean; 315 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.3 MiB/s wr, 920 op/s 2024-09-21T12:27:45.780 INFO:teuthology.orchestra.run.smithi078.stdout: 5 71123 13.93 MB/sec execute 23 sec latency 40.644 ms 2024-09-21T12:27:46.780 INFO:teuthology.orchestra.run.smithi078.stdout: 5 71334 13.58 MB/sec execute 24 sec latency 53.871 ms 2024-09-21T12:27:47.780 INFO:teuthology.orchestra.run.smithi078.stdout: 5 71851 13.30 MB/sec execute 25 sec latency 45.043 ms 2024-09-21T12:27:48.267 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:48 smithi078 ceph-mon[23681]: pgmap v20: 97 pgs: 97 active+clean; 315 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.8 MiB/s wr, 613 op/s 2024-09-21T12:27:48.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:48 smithi148 ceph-mon[25818]: pgmap v20: 97 pgs: 97 active+clean; 315 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.8 MiB/s wr, 613 op/s 2024-09-21T12:27:48.780 INFO:teuthology.orchestra.run.smithi078.stdout: 5 72487 13.47 MB/sec execute 26 sec latency 34.324 ms 2024-09-21T12:27:49.531 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:49 smithi078 ceph-mon[23681]: pgmap v21: 97 pgs: 97 active+clean; 322 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.7 MiB/s wr, 681 op/s 2024-09-21T12:27:49.531 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:49 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:49.531 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:49 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:49.531 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:49 smithi078 ceph-mon[23681]: Reconfiguring grafana.smithi078 (dependencies changed)... 2024-09-21T12:27:49.531 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:49 smithi078 ceph-mon[23681]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-21T12:27:49.780 INFO:teuthology.orchestra.run.smithi078.stdout: 5 73172 13.28 MB/sec execute 27 sec latency 40.113 ms 2024-09-21T12:27:49.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:49 smithi148 ceph-mon[25818]: pgmap v21: 97 pgs: 97 active+clean; 322 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.7 MiB/s wr, 681 op/s 2024-09-21T12:27:49.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:49 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:49.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:49 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:49.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:49 smithi148 ceph-mon[25818]: Reconfiguring grafana.smithi078 (dependencies changed)... 2024-09-21T12:27:49.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:49 smithi148 ceph-mon[25818]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-21T12:27:50.548 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:50 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:50.548 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:50 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:50.548 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:50 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T12:27:50.548 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:50 smithi078 ceph-mon[23681]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T12:27:50.548 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:50 smithi078 ceph-mon[23681]: Reconfiguring daemon grafana.smithi078 on smithi078 2024-09-21T12:27:50.782 INFO:teuthology.orchestra.run.smithi078.stdout: 5 73672 13.42 MB/sec execute 28 sec latency 24.780 ms 2024-09-21T12:27:50.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:50 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:50.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:50 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:50.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:50 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T12:27:50.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:50 smithi148 ceph-mon[25818]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T12:27:50.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:50 smithi148 ceph-mon[25818]: Reconfiguring daemon grafana.smithi078 on smithi078 2024-09-21T12:27:51.565 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:51 smithi078 ceph-mon[23681]: pgmap v22: 97 pgs: 97 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.1 MiB/s wr, 833 op/s 2024-09-21T12:27:51.780 INFO:teuthology.orchestra.run.smithi078.stdout: 5 74307 13.87 MB/sec execute 29 sec latency 24.275 ms 2024-09-21T12:27:51.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:51 smithi148 ceph-mon[25818]: pgmap v22: 97 pgs: 97 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.1 MiB/s wr, 833 op/s 2024-09-21T12:27:52.781 INFO:teuthology.orchestra.run.smithi078.stdout: 5 74767 14.01 MB/sec execute 30 sec latency 51.014 ms 2024-09-21T12:27:53.780 INFO:teuthology.orchestra.run.smithi078.stdout: 5 75069 13.67 MB/sec execute 31 sec latency 50.260 ms 2024-09-21T12:27:54.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:54 smithi078 ceph-mon[23681]: pgmap v23: 97 pgs: 97 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.4 MiB/s wr, 609 op/s 2024-09-21T12:27:54.423 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:54 smithi148 ceph-mon[25818]: pgmap v23: 97 pgs: 97 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.4 MiB/s wr, 609 op/s 2024-09-21T12:27:54.781 INFO:teuthology.orchestra.run.smithi078.stdout: 5 75462 13.40 MB/sec execute 32 sec latency 44.862 ms 2024-09-21T12:27:55.781 INFO:teuthology.orchestra.run.smithi078.stdout: 5 75879 13.32 MB/sec execute 33 sec latency 42.903 ms 2024-09-21T12:27:56.372 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:56 smithi078 ceph-mon[23681]: pgmap v24: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.8 MiB/s wr, 923 op/s 2024-09-21T12:27:56.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:56 smithi148 ceph-mon[25818]: pgmap v24: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.8 MiB/s wr, 923 op/s 2024-09-21T12:27:56.781 INFO:teuthology.orchestra.run.smithi078.stdout: 5 76462 13.29 MB/sec execute 34 sec latency 45.609 ms 2024-09-21T12:27:57.781 INFO:teuthology.orchestra.run.smithi078.stdout: 5 76939 13.10 MB/sec execute 35 sec latency 36.171 ms 2024-09-21T12:27:58.391 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:58 smithi078 ceph-mon[23681]: pgmap v25: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.7 MiB/s wr, 592 op/s 2024-09-21T12:27:58.391 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:58 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:58.391 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:58 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:58.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:58 smithi148 ceph-mon[25818]: pgmap v25: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.7 MiB/s wr, 592 op/s 2024-09-21T12:27:58.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:58 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:58.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:58 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:27:58.781 INFO:teuthology.orchestra.run.smithi078.stdout: 5 77376 13.23 MB/sec execute 36 sec latency 43.445 ms 2024-09-21T12:27:59.349 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:59 smithi078 ceph-mon[23681]: Reconfiguring node-exporter.smithi078 (dependencies changed)... 2024-09-21T12:27:59.349 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:59 smithi078 ceph-mon[23681]: Deploying daemon node-exporter.smithi078 on smithi078 2024-09-21T12:27:59.349 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:59 smithi078 ceph-mon[23681]: pgmap v26: 97 pgs: 97 active+clean; 320 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 8.8 MiB/s wr, 845 op/s 2024-09-21T12:27:59.349 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:27:59 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T12:27:59.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:59 smithi148 ceph-mon[25818]: Reconfiguring node-exporter.smithi078 (dependencies changed)... 2024-09-21T12:27:59.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:59 smithi148 ceph-mon[25818]: Deploying daemon node-exporter.smithi078 on smithi078 2024-09-21T12:27:59.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:59 smithi148 ceph-mon[25818]: pgmap v26: 97 pgs: 97 active+clean; 320 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 8.8 MiB/s wr, 845 op/s 2024-09-21T12:27:59.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:27:59 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T12:27:59.781 INFO:teuthology.orchestra.run.smithi078.stdout: 5 77966 13.53 MB/sec execute 37 sec latency 29.842 ms 2024-09-21T12:28:00.349 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-21T12:28:00.781 INFO:teuthology.orchestra.run.smithi078.stdout: 5 78321 13.57 MB/sec execute 38 sec latency 50.240 ms 2024-09-21T12:28:01.391 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:01 smithi078 ceph-mon[23681]: pgmap v27: 97 pgs: 97 active+clean; 314 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.1 MiB/s wr, 795 op/s 2024-09-21T12:28:01.392 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:01 smithi078 ceph-mon[23681]: from='client.14740 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:28:01.392 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:01 smithi078 ceph-mon[23681]: from='client.14744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:28:01.472 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T12:28:01.472 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 starting - - - - 2024-09-21T12:28:01.472 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (9m) 38s ago 9m 6400k - 17.2.0 e1d6a67b021e 4a209e5da5ce 2024-09-21T12:28:01.472 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi148 smithi148 running (8m) 37s ago 8m 6405k - 17.2.0 e1d6a67b021e 0fa5e612307f 2024-09-21T12:28:01.472 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 starting - - - - 2024-09-21T12:28:01.472 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.ytpzwh smithi078 running (3m) 38s ago 3m 34.5M - 17.2.0 e1d6a67b021e 5e9b109502a2 2024-09-21T12:28:01.472 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi148.uukezr smithi148 running (3m) 37s ago 3m 12.0M - 17.2.0 e1d6a67b021e b7e8aa0f3a6c 2024-09-21T12:28:01.472 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.ntwyan smithi078 *:9283 running (10m) 38s ago 10m 195M - 17.2.0 e1d6a67b021e caeba9628014 2024-09-21T12:28:01.473 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi148.oyiqmx smithi148 *:8443,9283 running (59s) 37s ago 8m 496M - 19.3.0-5104-g44178494 b415816db8b4 58467ce9a26b 2024-09-21T12:28:01.473 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (10m) 38s ago 10m 52.3M 2048M 17.2.0 e1d6a67b021e 8261b0b2c7e0 2024-09-21T12:28:01.473 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi148 smithi148 running (8m) 37s ago 8m 42.0M 2048M 17.2.0 e1d6a67b021e fda961db230f 2024-09-21T12:28:01.473 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.ydbhru smithi078 *:2049 running (2m) 38s ago 2m 239M - 4.0 e1d6a67b021e 7e5b0274ac38 2024-09-21T12:28:01.473 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi148.einuzt smithi148 *:2049 running (2m) 37s ago 2m 76.0M - 4.0 e1d6a67b021e 76a8b0eabede 2024-09-21T12:28:01.473 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (8m) 38s ago 8m 17.5M - 1.3.1 1dbe0e931976 e7ca2ce65161 2024-09-21T12:28:01.473 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi148 smithi148 *:9100 running (8m) 37s ago 8m 21.8M - 1.3.1 1dbe0e931976 1474152580ab 2024-09-21T12:28:01.473 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (7m) 38s ago 7m 265M 1936M 17.2.0 e1d6a67b021e d4d5e9977070 2024-09-21T12:28:01.473 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (7m) 38s ago 7m 196M 1936M 17.2.0 e1d6a67b021e b9a4b952233b 2024-09-21T12:28:01.473 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (6m) 38s ago 6m 171M 1936M 17.2.0 e1d6a67b021e 829fb8c2d827 2024-09-21T12:28:01.473 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (6m) 38s ago 6m 214M 1936M 17.2.0 e1d6a67b021e ef942c8556d0 2024-09-21T12:28:01.474 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi148 running (5m) 37s ago 5m 230M 2704M 17.2.0 e1d6a67b021e 62eefdd572c0 2024-09-21T12:28:01.474 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi148 running (5m) 37s ago 5m 234M 2704M 17.2.0 e1d6a67b021e 7b44be78baca 2024-09-21T12:28:01.474 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi148 running (5m) 37s ago 5m 224M 2704M 17.2.0 e1d6a67b021e 3d23057910ea 2024-09-21T12:28:01.474 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi148 running (4m) 37s ago 4m 314M 2704M 17.2.0 e1d6a67b021e f9e4db9d35fa 2024-09-21T12:28:01.474 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (8m) 38s ago 8m 63.8M - 2.33.4 514e6a882f6e cdcd63cf2477 2024-09-21T12:28:01.781 INFO:teuthology.orchestra.run.smithi078.stdout: 5 78661 13.33 MB/sec execute 39 sec latency 44.511 ms 2024-09-21T12:28:01.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:01 smithi148 ceph-mon[25818]: pgmap v27: 97 pgs: 97 active+clean; 314 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.1 MiB/s wr, 795 op/s 2024-09-21T12:28:01.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:01 smithi148 ceph-mon[25818]: from='client.14740 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:28:01.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:01 smithi148 ceph-mon[25818]: from='client.14744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:28:02.158 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:28:02.158 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-21T12:28:02.158 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:28:02.158 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:28:02.158 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-21T12:28:02.158 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-21T12:28:02.158 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 1 2024-09-21T12:28:02.158 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:28:02.158 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-21T12:28:02.158 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T12:28:02.158 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:28:02.159 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-21T12:28:02.159 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:28:02.159 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:28:02.159 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-21T12:28:02.159 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-21T12:28:02.159 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 1 2024-09-21T12:28:02.159 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T12:28:02.159 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:28:02.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:02 smithi078 ceph-mon[23681]: from='client.14748 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:28:02.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:02 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/2459902680' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T12:28:02.671 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:28:02.671 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5", 2024-09-21T12:28:02.671 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-21T12:28:02.672 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T12:28:02.672 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-09-21T12:28:02.672 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "1/23 daemons upgraded", 2024-09-21T12:28:02.672 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "", 2024-09-21T12:28:02.672 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-21T12:28:02.672 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:28:02.781 INFO:teuthology.orchestra.run.smithi078.stdout: 5 79096 13.13 MB/sec execute 40 sec latency 39.916 ms 2024-09-21T12:28:02.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:02 smithi148 ceph-mon[25818]: from='client.14748 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:28:02.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:02 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/2459902680' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T12:28:03.245 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-21T12:28:03.781 INFO:teuthology.orchestra.run.smithi078.stdout: 5 79408 13.05 MB/sec execute 41 sec latency 47.413 ms 2024-09-21T12:28:04.419 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:04 smithi078 ceph-mon[23681]: pgmap v28: 97 pgs: 97 active+clean; 314 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.7 MiB/s wr, 585 op/s 2024-09-21T12:28:04.419 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:04 smithi078 ceph-mon[23681]: from='client.14756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:28:04.419 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:04 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/688292228' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T12:28:04.422 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:04 smithi148 ceph-mon[25818]: pgmap v28: 97 pgs: 97 active+clean; 314 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.7 MiB/s wr, 585 op/s 2024-09-21T12:28:04.423 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:04 smithi148 ceph-mon[25818]: from='client.14756 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:28:04.423 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:04 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/688292228' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T12:28:04.781 INFO:teuthology.orchestra.run.smithi078.stdout: 5 80009 13.03 MB/sec execute 42 sec latency 40.921 ms 2024-09-21T12:28:05.781 INFO:teuthology.orchestra.run.smithi078.stdout: 5 80555 12.97 MB/sec execute 43 sec latency 27.685 ms 2024-09-21T12:28:06.179 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:06 smithi078 ceph-mon[23681]: pgmap v29: 97 pgs: 97 active+clean; 327 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.7 MiB/s wr, 841 op/s 2024-09-21T12:28:06.179 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:06 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:06.179 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:06 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:06.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:06 smithi148 ceph-mon[25818]: pgmap v29: 97 pgs: 97 active+clean; 327 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.7 MiB/s wr, 841 op/s 2024-09-21T12:28:06.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:06 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:06.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:06 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:06.781 INFO:teuthology.orchestra.run.smithi078.stdout: 5 80938 13.02 MB/sec execute 44 sec latency 41.860 ms 2024-09-21T12:28:07.220 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:07 smithi078 ceph-mon[23681]: Reconfiguring prometheus.smithi078 (dependencies changed)... 2024-09-21T12:28:07.220 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:07 smithi078 ceph-mon[23681]: Deploying daemon prometheus.smithi078 on smithi078 2024-09-21T12:28:07.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:07 smithi148 ceph-mon[25818]: Reconfiguring prometheus.smithi078 (dependencies changed)... 2024-09-21T12:28:07.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:07 smithi148 ceph-mon[25818]: Deploying daemon prometheus.smithi078 on smithi078 2024-09-21T12:28:07.781 INFO:teuthology.orchestra.run.smithi078.stdout: 5 81425 13.20 MB/sec execute 45 sec latency 66.836 ms 2024-09-21T12:28:08.245 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:08 smithi078 ceph-mon[23681]: pgmap v30: 97 pgs: 97 active+clean; 327 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.3 MiB/s wr, 527 op/s 2024-09-21T12:28:08.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:08 smithi148 ceph-mon[25818]: pgmap v30: 97 pgs: 97 active+clean; 327 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.3 MiB/s wr, 527 op/s 2024-09-21T12:28:08.781 INFO:teuthology.orchestra.run.smithi078.stdout: 5 81763 13.20 MB/sec execute 46 sec latency 44.889 ms 2024-09-21T12:28:09.260 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:09 smithi078 ceph-mon[23681]: pgmap v31: 97 pgs: 97 active+clean; 330 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.9 MiB/s wr, 780 op/s 2024-09-21T12:28:09.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:09 smithi148 ceph-mon[25818]: pgmap v31: 97 pgs: 97 active+clean; 330 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.9 MiB/s wr, 780 op/s 2024-09-21T12:28:09.781 INFO:teuthology.orchestra.run.smithi078.stdout: 5 82131 13.07 MB/sec execute 47 sec latency 43.839 ms 2024-09-21T12:28:10.781 INFO:teuthology.orchestra.run.smithi078.stdout: 5 82609 12.93 MB/sec execute 48 sec latency 33.263 ms 2024-09-21T12:28:11.523 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:11 smithi078 ceph-mon[23681]: pgmap v32: 97 pgs: 97 active+clean; 326 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 6.0 MiB/s wr, 542 op/s 2024-09-21T12:28:11.781 INFO:teuthology.orchestra.run.smithi078.stdout: 5 83224 12.96 MB/sec execute 49 sec latency 41.195 ms 2024-09-21T12:28:11.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:11 smithi148 ceph-mon[25818]: pgmap v32: 97 pgs: 97 active+clean; 326 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 6.0 MiB/s wr, 542 op/s 2024-09-21T12:28:12.781 INFO:teuthology.orchestra.run.smithi078.stdout: 5 83787 12.90 MB/sec execute 50 sec latency 44.165 ms 2024-09-21T12:28:13.781 INFO:teuthology.orchestra.run.smithi078.stdout: 5 84273 12.94 MB/sec execute 51 sec latency 53.207 ms 2024-09-21T12:28:14.331 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:14 smithi078 ceph-mon[23681]: pgmap v33: 97 pgs: 97 active+clean; 326 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.8 MiB/s wr, 524 op/s 2024-09-21T12:28:14.331 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:14 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T12:28:14.422 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:14 smithi148 ceph-mon[25818]: pgmap v33: 97 pgs: 97 active+clean; 326 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.8 MiB/s wr, 524 op/s 2024-09-21T12:28:14.423 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:14 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T12:28:14.781 INFO:teuthology.orchestra.run.smithi078.stdout: 5 84825 13.12 MB/sec execute 52 sec latency 32.564 ms 2024-09-21T12:28:15.782 INFO:teuthology.orchestra.run.smithi078.stdout: 5 85243 13.24 MB/sec execute 53 sec latency 72.036 ms 2024-09-21T12:28:16.388 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:16 smithi078 ceph-mon[23681]: pgmap v34: 97 pgs: 97 active+clean; 316 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 813 op/s 2024-09-21T12:28:16.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:16 smithi148 ceph-mon[25818]: pgmap v34: 97 pgs: 97 active+clean; 316 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 813 op/s 2024-09-21T12:28:16.782 INFO:teuthology.orchestra.run.smithi078.stdout: 5 85632 13.14 MB/sec execute 54 sec latency 33.488 ms 2024-09-21T12:28:17.782 INFO:teuthology.orchestra.run.smithi078.stdout: 5 86045 13.00 MB/sec execute 55 sec latency 32.572 ms 2024-09-21T12:28:18.410 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:18 smithi078 ceph-mon[23681]: pgmap v35: 97 pgs: 97 active+clean; 316 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.3 MiB/s wr, 557 op/s 2024-09-21T12:28:18.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:18 smithi148 ceph-mon[25818]: pgmap v35: 97 pgs: 97 active+clean; 316 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.3 MiB/s wr, 557 op/s 2024-09-21T12:28:18.782 INFO:teuthology.orchestra.run.smithi078.stdout: 5 86435 12.95 MB/sec execute 56 sec latency 41.682 ms 2024-09-21T12:28:19.782 INFO:teuthology.orchestra.run.smithi078.stdout: 5 86904 12.88 MB/sec execute 57 sec latency 47.975 ms 2024-09-21T12:28:20.212 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:20 smithi078 ceph-mon[23681]: pgmap v36: 97 pgs: 97 active+clean; 331 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.7 MiB/s wr, 838 op/s 2024-09-21T12:28:20.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:20 smithi148 ceph-mon[25818]: pgmap v36: 97 pgs: 97 active+clean; 331 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.7 MiB/s wr, 838 op/s 2024-09-21T12:28:20.782 INFO:teuthology.orchestra.run.smithi078.stdout: 5 87370 12.79 MB/sec execute 58 sec latency 37.245 ms 2024-09-21T12:28:21.782 INFO:teuthology.orchestra.run.smithi078.stdout: 5 87855 12.87 MB/sec execute 59 sec latency 43.186 ms 2024-09-21T12:28:22.243 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:22 smithi078 ceph-mon[23681]: pgmap v37: 97 pgs: 97 active+clean; 331 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.1 MiB/s wr, 584 op/s 2024-09-21T12:28:22.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:22 smithi148 ceph-mon[25818]: pgmap v37: 97 pgs: 97 active+clean; 331 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.1 MiB/s wr, 584 op/s 2024-09-21T12:28:22.782 INFO:teuthology.orchestra.run.smithi078.stdout: 5 88357 12.96 MB/sec execute 60 sec latency 54.966 ms 2024-09-21T12:28:23.782 INFO:teuthology.orchestra.run.smithi078.stdout: 5 88819 13.06 MB/sec execute 61 sec latency 43.168 ms 2024-09-21T12:28:24.204 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:24 smithi078 ceph-mon[23681]: pgmap v38: 97 pgs: 97 active+clean; 331 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.8 MiB/s wr, 570 op/s 2024-09-21T12:28:24.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:24 smithi148 ceph-mon[25818]: pgmap v38: 97 pgs: 97 active+clean; 331 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.8 MiB/s wr, 570 op/s 2024-09-21T12:28:24.782 INFO:teuthology.orchestra.run.smithi078.stdout: 5 89177 12.98 MB/sec execute 62 sec latency 51.193 ms 2024-09-21T12:28:25.782 INFO:teuthology.orchestra.run.smithi078.stdout: 5 89478 12.83 MB/sec execute 63 sec latency 52.286 ms 2024-09-21T12:28:26.239 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:26 smithi078 ceph-mon[23681]: pgmap v39: 97 pgs: 97 active+clean; 330 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 835 op/s 2024-09-21T12:28:26.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:26 smithi148 ceph-mon[25818]: pgmap v39: 97 pgs: 97 active+clean; 330 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 835 op/s 2024-09-21T12:28:26.782 INFO:teuthology.orchestra.run.smithi078.stdout: 5 89750 12.70 MB/sec execute 64 sec latency 94.952 ms 2024-09-21T12:28:27.782 INFO:teuthology.orchestra.run.smithi078.stdout: 5 90127 12.68 MB/sec execute 65 sec latency 61.832 ms 2024-09-21T12:28:28.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:28 smithi148 ceph-mon[25818]: pgmap v40: 97 pgs: 97 active+clean; 330 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.1 MiB/s wr, 546 op/s 2024-09-21T12:28:28.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:28 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:28.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:28 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:28.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:28 smithi148 ceph-mon[25818]: Reconfiguring node-exporter.smithi148 (dependencies changed)... 2024-09-21T12:28:28.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:28 smithi148 ceph-mon[25818]: Deploying daemon node-exporter.smithi148 on smithi148 2024-09-21T12:28:28.305 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:28 smithi078 ceph-mon[23681]: pgmap v40: 97 pgs: 97 active+clean; 330 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.1 MiB/s wr, 546 op/s 2024-09-21T12:28:28.305 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:28 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:28.305 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:28 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:28.305 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:28 smithi078 ceph-mon[23681]: Reconfiguring node-exporter.smithi148 (dependencies changed)... 2024-09-21T12:28:28.305 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:28 smithi078 ceph-mon[23681]: Deploying daemon node-exporter.smithi148 on smithi148 2024-09-21T12:28:28.782 INFO:teuthology.orchestra.run.smithi078.stdout: 5 90641 12.59 MB/sec execute 66 sec latency 66.530 ms 2024-09-21T12:28:29.324 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:29 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T12:28:29.449 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:29 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T12:28:29.782 INFO:teuthology.orchestra.run.smithi078.stdout: 5 90987 12.53 MB/sec execute 67 sec latency 54.074 ms 2024-09-21T12:28:30.176 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:30 smithi078 ceph-mon[23681]: pgmap v41: 97 pgs: 97 active+clean; 322 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 776 op/s 2024-09-21T12:28:30.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:30 smithi148 ceph-mon[25818]: pgmap v41: 97 pgs: 97 active+clean; 322 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 776 op/s 2024-09-21T12:28:30.782 INFO:teuthology.orchestra.run.smithi078.stdout: 5 91396 12.55 MB/sec execute 68 sec latency 51.618 ms 2024-09-21T12:28:31.782 INFO:teuthology.orchestra.run.smithi078.stdout: 5 91851 12.60 MB/sec execute 69 sec latency 56.428 ms 2024-09-21T12:28:32.323 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:32 smithi148 ceph-mon[25818]: pgmap v42: 97 pgs: 97 active+clean; 322 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 4.8 MiB/s wr, 494 op/s 2024-09-21T12:28:32.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:32 smithi078 ceph-mon[23681]: pgmap v42: 97 pgs: 97 active+clean; 322 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 4.8 MiB/s wr, 494 op/s 2024-09-21T12:28:32.782 INFO:teuthology.orchestra.run.smithi078.stdout: 5 92322 12.74 MB/sec execute 70 sec latency 47.590 ms 2024-09-21T12:28:33.782 INFO:teuthology.orchestra.run.smithi078.stdout: 5 92543 12.66 MB/sec execute 71 sec latency 55.424 ms 2024-09-21T12:28:33.802 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-21T12:28:33.918 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:33 smithi078 ceph-mon[23681]: pgmap v43: 97 pgs: 97 active+clean; 322 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 4.8 MiB/s wr, 494 op/s 2024-09-21T12:28:33.918 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:33 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:33.918 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:33 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:33.918 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:33 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T12:28:33.918 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:33 smithi078 ceph-mon[23681]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T12:28:33.918 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:33 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi078.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T12:28:33.919 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:33 smithi078 ceph-mon[23681]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi078.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T12:28:33.919 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:33 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:33.919 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:33 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T12:28:33.919 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:33 smithi078 ceph-mon[23681]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T12:28:33.919 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:33 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi078.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T12:28:33.919 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:33 smithi078 ceph-mon[23681]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi078.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T12:28:33.919 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:33 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:33.919 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:33 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T12:28:33.919 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:33 smithi078 ceph-mon[23681]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T12:28:33.919 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:33 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi078.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T12:28:33.919 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:33 smithi078 ceph-mon[23681]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi078.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T12:28:33.919 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:33 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:33.920 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:33 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T12:28:33.920 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:33 smithi078 ceph-mon[23681]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T12:28:33.920 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:33 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:28:33.920 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:33 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T12:28:34.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:33 smithi148 ceph-mon[25818]: pgmap v43: 97 pgs: 97 active+clean; 322 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 4.8 MiB/s wr, 494 op/s 2024-09-21T12:28:34.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:33 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:34.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:33 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:34.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:33 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T12:28:34.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:33 smithi148 ceph-mon[25818]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T12:28:34.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:33 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi078.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T12:28:34.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:33 smithi148 ceph-mon[25818]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi078.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T12:28:34.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:33 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:34.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:33 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T12:28:34.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:33 smithi148 ceph-mon[25818]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T12:28:34.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:33 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi078.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T12:28:34.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:33 smithi148 ceph-mon[25818]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi078.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T12:28:34.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:33 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:34.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:33 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T12:28:34.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:33 smithi148 ceph-mon[25818]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T12:28:34.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:33 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi078.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T12:28:34.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:33 smithi148 ceph-mon[25818]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi078.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T12:28:34.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:33 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:34.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:33 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T12:28:34.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:33 smithi148 ceph-mon[25818]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T12:28:34.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:33 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:28:34.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:33 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T12:28:34.763 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T12:28:34.763 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 starting - - - - 2024-09-21T12:28:34.763 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (9m) 71s ago 9m 6400k - 17.2.0 e1d6a67b021e 4a209e5da5ce 2024-09-21T12:28:34.763 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi148 smithi148 running (9m) 70s ago 9m 6405k - 17.2.0 e1d6a67b021e 0fa5e612307f 2024-09-21T12:28:34.763 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 starting - - - - 2024-09-21T12:28:34.763 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.ytpzwh smithi078 running (3m) 71s ago 3m 34.5M - 17.2.0 e1d6a67b021e 5e9b109502a2 2024-09-21T12:28:34.763 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi148.uukezr smithi148 running (3m) 70s ago 3m 12.0M - 17.2.0 e1d6a67b021e b7e8aa0f3a6c 2024-09-21T12:28:34.763 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.ntwyan smithi078 *:9283 running (11m) 71s ago 11m 195M - 17.2.0 e1d6a67b021e caeba9628014 2024-09-21T12:28:34.763 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi148.oyiqmx smithi148 *:8443,9283 running (93s) 70s ago 9m 496M - 19.3.0-5104-g44178494 b415816db8b4 58467ce9a26b 2024-09-21T12:28:34.764 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (11m) 71s ago 11m 52.3M 2048M 17.2.0 e1d6a67b021e 8261b0b2c7e0 2024-09-21T12:28:34.764 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi148 smithi148 running (9m) 70s ago 9m 42.0M 2048M 17.2.0 e1d6a67b021e fda961db230f 2024-09-21T12:28:34.764 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.ydbhru smithi078 *:2049 running (3m) 71s ago 3m 239M - 4.0 e1d6a67b021e 7e5b0274ac38 2024-09-21T12:28:34.764 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi148.einuzt smithi148 *:2049 running (3m) 70s ago 3m 76.0M - 4.0 e1d6a67b021e 76a8b0eabede 2024-09-21T12:28:34.764 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 starting - - - - 2024-09-21T12:28:34.764 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi148 smithi148 *:9100 starting - - - - 2024-09-21T12:28:34.764 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (8m) 71s ago 8m 265M 1936M 17.2.0 e1d6a67b021e d4d5e9977070 2024-09-21T12:28:34.764 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (7m) 71s ago 7m 196M 1936M 17.2.0 e1d6a67b021e b9a4b952233b 2024-09-21T12:28:34.764 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (7m) 71s ago 7m 171M 1936M 17.2.0 e1d6a67b021e 829fb8c2d827 2024-09-21T12:28:34.764 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (6m) 71s ago 6m 214M 1936M 17.2.0 e1d6a67b021e ef942c8556d0 2024-09-21T12:28:34.764 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi148 running (6m) 70s ago 6m 230M 2704M 17.2.0 e1d6a67b021e 62eefdd572c0 2024-09-21T12:28:34.764 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi148 running (6m) 70s ago 6m 234M 2704M 17.2.0 e1d6a67b021e 7b44be78baca 2024-09-21T12:28:34.764 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi148 running (5m) 70s ago 5m 224M 2704M 17.2.0 e1d6a67b021e 3d23057910ea 2024-09-21T12:28:34.765 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi148 running (5m) 70s ago 5m 314M 2704M 17.2.0 e1d6a67b021e f9e4db9d35fa 2024-09-21T12:28:34.765 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 starting - - - - 2024-09-21T12:28:34.782 INFO:teuthology.orchestra.run.smithi078.stdout: 5 92868 12.53 MB/sec execute 72 sec latency 41.384 ms 2024-09-21T12:28:34.931 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:34 smithi078 ceph-mon[23681]: Upgrade: Need to upgrade myself (mgr.smithi148.oyiqmx) 2024-09-21T12:28:34.931 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:34 smithi078 ceph-mon[23681]: Upgrade: Need to upgrade myself (mgr.smithi148.oyiqmx) 2024-09-21T12:28:34.931 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:34 smithi078 ceph-mon[23681]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T12:28:34.931 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:34 smithi078 ceph-mon[23681]: from='client.14764 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:28:34.931 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:34 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T12:28:35.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:34 smithi148 ceph-mon[25818]: Upgrade: Need to upgrade myself (mgr.smithi148.oyiqmx) 2024-09-21T12:28:35.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:34 smithi148 ceph-mon[25818]: Upgrade: Need to upgrade myself (mgr.smithi148.oyiqmx) 2024-09-21T12:28:35.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:34 smithi148 ceph-mon[25818]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T12:28:35.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:34 smithi148 ceph-mon[25818]: from='client.14764 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:28:35.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:34 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T12:28:35.389 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:28:35.389 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-21T12:28:35.389 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:28:35.389 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:28:35.389 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-21T12:28:35.389 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-21T12:28:35.389 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 1 2024-09-21T12:28:35.389 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:28:35.389 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-21T12:28:35.389 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T12:28:35.389 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:28:35.389 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-21T12:28:35.390 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:28:35.390 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:28:35.390 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-21T12:28:35.390 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-21T12:28:35.390 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 1 2024-09-21T12:28:35.390 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T12:28:35.390 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:28:35.782 INFO:teuthology.orchestra.run.smithi078.stdout: 5 93185 12.43 MB/sec execute 73 sec latency 40.224 ms 2024-09-21T12:28:35.860 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:28:35.861 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5", 2024-09-21T12:28:35.861 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-21T12:28:35.861 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T12:28:35.861 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-09-21T12:28:35.861 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "1/23 daemons upgraded", 2024-09-21T12:28:35.861 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading mgr daemons", 2024-09-21T12:28:35.861 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-21T12:28:35.861 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:28:35.943 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:35 smithi078 ceph-mon[23681]: from='client.14768 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:28:35.943 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:35 smithi078 ceph-mon[23681]: pgmap v44: 97 pgs: 97 active+clean; 329 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 732 op/s 2024-09-21T12:28:35.943 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:35 smithi078 ceph-mon[23681]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T12:28:35.943 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:35 smithi078 ceph-mon[23681]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:28:35.943 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:35 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/2977202750' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T12:28:36.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:35 smithi148 ceph-mon[25818]: from='client.14768 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:28:36.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:35 smithi148 ceph-mon[25818]: pgmap v44: 97 pgs: 97 active+clean; 329 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 732 op/s 2024-09-21T12:28:36.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:35 smithi148 ceph-mon[25818]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T12:28:36.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:35 smithi148 ceph-mon[25818]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:28:36.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:35 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/2977202750' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T12:28:36.460 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-21T12:28:36.782 INFO:teuthology.orchestra.run.smithi078.stdout: 5 93496 12.38 MB/sec execute 74 sec latency 99.335 ms 2024-09-21T12:28:36.957 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:36 smithi078 ceph-mon[23681]: from='client.14780 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:28:36.957 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:36 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/1853070639' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T12:28:37.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:36 smithi148 ceph-mon[25818]: from='client.14780 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:28:37.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:36 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/1853070639' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T12:28:37.783 INFO:teuthology.orchestra.run.smithi078.stdout: 5 93937 12.33 MB/sec execute 75 sec latency 60.423 ms 2024-09-21T12:28:38.219 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:38 smithi078 ceph-mon[23681]: pgmap v45: 97 pgs: 97 active+clean; 329 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 467 op/s 2024-09-21T12:28:38.220 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:38 smithi078 ceph-mon[23681]: Upgrade: Updating mgr.smithi078.ntwyan 2024-09-21T12:28:38.220 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:38 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:38.220 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:38 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.ntwyan", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T12:28:38.220 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:38 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.ntwyan", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T12:28:38.220 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:38 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T12:28:38.220 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:38 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:28:38.220 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:38 smithi078 ceph-mon[23681]: Deploying daemon mgr.smithi078.ntwyan on smithi078 2024-09-21T12:28:38.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:38 smithi148 ceph-mon[25818]: pgmap v45: 97 pgs: 97 active+clean; 329 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 467 op/s 2024-09-21T12:28:38.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:38 smithi148 ceph-mon[25818]: Upgrade: Updating mgr.smithi078.ntwyan 2024-09-21T12:28:38.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:38 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:38.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:38 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.ntwyan", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T12:28:38.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:38 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.ntwyan", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T12:28:38.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:38 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T12:28:38.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:38 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:28:38.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:38 smithi148 ceph-mon[25818]: Deploying daemon mgr.smithi078.ntwyan on smithi078 2024-09-21T12:28:38.783 INFO:teuthology.orchestra.run.smithi078.stdout: 5 94488 12.29 MB/sec execute 76 sec latency 46.090 ms 2024-09-21T12:28:39.783 INFO:teuthology.orchestra.run.smithi078.stdout: 5 94911 12.29 MB/sec execute 77 sec latency 62.484 ms 2024-09-21T12:28:40.248 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:40 smithi078 ceph-mon[23681]: pgmap v46: 97 pgs: 97 active+clean; 321 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 688 op/s 2024-09-21T12:28:40.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:40 smithi148 ceph-mon[25818]: pgmap v46: 97 pgs: 97 active+clean; 321 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 688 op/s 2024-09-21T12:28:40.783 INFO:teuthology.orchestra.run.smithi078.stdout: 5 95311 12.35 MB/sec execute 78 sec latency 88.019 ms 2024-09-21T12:28:41.783 INFO:teuthology.orchestra.run.smithi078.stdout: 5 95766 12.46 MB/sec execute 79 sec latency 37.219 ms 2024-09-21T12:28:42.286 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:42 smithi078 ceph-mon[23681]: pgmap v47: 97 pgs: 97 active+clean; 321 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 459 op/s 2024-09-21T12:28:42.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:42 smithi148 ceph-mon[25818]: pgmap v47: 97 pgs: 97 active+clean; 321 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 459 op/s 2024-09-21T12:28:42.783 INFO:teuthology.orchestra.run.smithi078.stdout: 5 96023 12.40 MB/sec execute 80 sec latency 63.380 ms 2024-09-21T12:28:43.783 INFO:teuthology.orchestra.run.smithi078.stdout: 5 96301 12.31 MB/sec execute 81 sec latency 60.072 ms 2024-09-21T12:28:44.319 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:44 smithi078 ceph-mon[23681]: pgmap v48: 97 pgs: 97 active+clean; 321 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 459 op/s 2024-09-21T12:28:44.319 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:44 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:44.319 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:44 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:44.319 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:44 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:28:44.319 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:44 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T12:28:44.319 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:44 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:44.319 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:44 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:44.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:44 smithi148 ceph-mon[25818]: pgmap v48: 97 pgs: 97 active+clean; 321 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 459 op/s 2024-09-21T12:28:44.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:44 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:44.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:44 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:44.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:44 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:28:44.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:44 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T12:28:44.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:44 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:44.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:44 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:44.783 INFO:teuthology.orchestra.run.smithi078.stdout: 5 96684 12.21 MB/sec execute 82 sec latency 35.873 ms 2024-09-21T12:28:45.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:45 smithi148 ceph-mon[25818]: pgmap v49: 97 pgs: 97 active+clean; 335 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 7.4 MiB/s wr, 685 op/s 2024-09-21T12:28:45.337 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:45 smithi078 ceph-mon[23681]: pgmap v49: 97 pgs: 97 active+clean; 335 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 7.4 MiB/s wr, 685 op/s 2024-09-21T12:28:45.783 INFO:teuthology.orchestra.run.smithi078.stdout: 5 97007 12.19 MB/sec execute 83 sec latency 76.823 ms 2024-09-21T12:28:46.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:46 smithi148 ceph-mon[25818]: mgrmap e28: smithi148.oyiqmx(active, since 93s), standbys: smithi078.ntwyan 2024-09-21T12:28:46.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:46 smithi078 ceph-mon[23681]: mgrmap e28: smithi148.oyiqmx(active, since 93s), standbys: smithi078.ntwyan 2024-09-21T12:28:46.783 INFO:teuthology.orchestra.run.smithi078.stdout: 5 97313 12.13 MB/sec execute 84 sec latency 57.790 ms 2024-09-21T12:28:47.783 INFO:teuthology.orchestra.run.smithi078.stdout: 5 97757 12.03 MB/sec execute 85 sec latency 60.466 ms 2024-09-21T12:28:48.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:48 smithi078 ceph-mon[23681]: pgmap v50: 97 pgs: 97 active+clean; 335 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 447 op/s 2024-09-21T12:28:48.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:48 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:48.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:48 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:48.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:48 smithi148 ceph-mon[25818]: pgmap v50: 97 pgs: 97 active+clean; 335 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 447 op/s 2024-09-21T12:28:48.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:48 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:48.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:48 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:48.783 INFO:teuthology.orchestra.run.smithi078.stdout: 5 98049 11.98 MB/sec execute 86 sec latency 61.337 ms 2024-09-21T12:28:49.783 INFO:teuthology.orchestra.run.smithi078.stdout: 5 98307 11.92 MB/sec execute 87 sec latency 53.738 ms 2024-09-21T12:28:50.425 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:50 smithi078 ceph-mon[23681]: pgmap v51: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.2 MiB/s wr, 666 op/s 2024-09-21T12:28:50.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:50 smithi148 ceph-mon[25818]: pgmap v51: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.2 MiB/s wr, 666 op/s 2024-09-21T12:28:50.783 INFO:teuthology.orchestra.run.smithi078.stdout: 5 98649 11.96 MB/sec execute 88 sec latency 69.158 ms 2024-09-21T12:28:51.462 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:51 smithi078 ceph-mon[23681]: Standby manager daemon smithi078.ntwyan restarted 2024-09-21T12:28:51.462 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:51 smithi078 ceph-mon[23681]: Standby manager daemon smithi078.ntwyan started 2024-09-21T12:28:51.462 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:51 smithi078 ceph-mon[23681]: from='mgr.? 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.ntwyan/crt"}]: dispatch 2024-09-21T12:28:51.462 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:51 smithi078 ceph-mon[23681]: from='mgr.? 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T12:28:51.462 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:51 smithi078 ceph-mon[23681]: from='mgr.? 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.ntwyan/key"}]: dispatch 2024-09-21T12:28:51.462 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:51 smithi078 ceph-mon[23681]: from='mgr.? 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T12:28:51.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:51 smithi148 ceph-mon[25818]: Standby manager daemon smithi078.ntwyan restarted 2024-09-21T12:28:51.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:51 smithi148 ceph-mon[25818]: Standby manager daemon smithi078.ntwyan started 2024-09-21T12:28:51.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:51 smithi148 ceph-mon[25818]: from='mgr.? 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.ntwyan/crt"}]: dispatch 2024-09-21T12:28:51.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:51 smithi148 ceph-mon[25818]: from='mgr.? 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T12:28:51.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:51 smithi148 ceph-mon[25818]: from='mgr.? 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.ntwyan/key"}]: dispatch 2024-09-21T12:28:51.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:51 smithi148 ceph-mon[25818]: from='mgr.? 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T12:28:51.783 INFO:teuthology.orchestra.run.smithi078.stdout: 5 99000 11.96 MB/sec execute 89 sec latency 85.749 ms 2024-09-21T12:28:52.467 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:52 smithi078 ceph-mon[23681]: pgmap v52: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.6 MiB/s wr, 445 op/s 2024-09-21T12:28:52.468 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:52 smithi078 ceph-mon[23681]: mgrmap e29: smithi148.oyiqmx(active, since 99s), standbys: smithi078.ntwyan 2024-09-21T12:28:52.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:52 smithi148 ceph-mon[25818]: pgmap v52: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.6 MiB/s wr, 445 op/s 2024-09-21T12:28:52.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:52 smithi148 ceph-mon[25818]: mgrmap e29: smithi148.oyiqmx(active, since 99s), standbys: smithi078.ntwyan 2024-09-21T12:28:52.783 INFO:teuthology.orchestra.run.smithi078.stdout: 5 99395 12.03 MB/sec execute 90 sec latency 55.512 ms 2024-09-21T12:28:53.500 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:53 smithi078 ceph-mon[23681]: pgmap v53: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.6 MiB/s wr, 445 op/s 2024-09-21T12:28:53.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:53 smithi148 ceph-mon[25818]: pgmap v53: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.6 MiB/s wr, 445 op/s 2024-09-21T12:28:53.783 INFO:teuthology.orchestra.run.smithi078.stdout: 5 99608 11.95 MB/sec execute 91 sec latency 62.158 ms 2024-09-21T12:28:54.783 INFO:teuthology.orchestra.run.smithi078.stdout: 5 99933 11.88 MB/sec execute 92 sec latency 86.630 ms 2024-09-21T12:28:55.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:55 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:55.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:55 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:55.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:55 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:55.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:55 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:55.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:55 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:55.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:55 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:55.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:55 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:55.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:55 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:28:55.783 INFO:teuthology.orchestra.run.smithi078.stdout: 5 100162 11.78 MB/sec execute 93 sec latency 57.104 ms 2024-09-21T12:28:56.247 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:56 smithi078 ceph-mon[23681]: pgmap v54: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 7.1 MiB/s wr, 633 op/s 2024-09-21T12:28:56.531 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:56 smithi148 ceph-mon[25818]: pgmap v54: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 7.1 MiB/s wr, 633 op/s 2024-09-21T12:28:56.783 INFO:teuthology.orchestra.run.smithi078.stdout: 5 100404 11.70 MB/sec execute 94 sec latency 137.123 ms 2024-09-21T12:28:57.783 INFO:teuthology.orchestra.run.smithi078.stdout: 5 100781 11.70 MB/sec execute 95 sec latency 57.601 ms 2024-09-21T12:28:58.282 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:58 smithi078 ceph-mon[23681]: pgmap v55: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 408 op/s 2024-09-21T12:28:58.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:58 smithi148 ceph-mon[25818]: pgmap v55: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 408 op/s 2024-09-21T12:28:58.783 INFO:teuthology.orchestra.run.smithi078.stdout: 5 101228 11.64 MB/sec execute 96 sec latency 45.641 ms 2024-09-21T12:28:59.333 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:28:59 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T12:28:59.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:28:59 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T12:28:59.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 101606 11.61 MB/sec execute 97 sec latency 77.648 ms 2024-09-21T12:29:00.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:00 smithi078 ceph-mon[23681]: pgmap v56: 97 pgs: 97 active+clean; 328 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 5.7 MiB/s wr, 598 op/s 2024-09-21T12:29:00.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:00 smithi148 ceph-mon[25818]: pgmap v56: 97 pgs: 97 active+clean; 328 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 5.7 MiB/s wr, 598 op/s 2024-09-21T12:29:00.783 INFO:teuthology.orchestra.run.smithi078.stdout: 5 102034 11.63 MB/sec execute 98 sec latency 48.826 ms 2024-09-21T12:29:01.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:01 smithi078 ceph-mon[23681]: pgmap v57: 97 pgs: 97 active+clean; 328 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.0 MiB/s wr, 378 op/s 2024-09-21T12:29:01.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:01 smithi148 ceph-mon[25818]: pgmap v57: 97 pgs: 97 active+clean; 328 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.0 MiB/s wr, 378 op/s 2024-09-21T12:29:01.783 INFO:teuthology.orchestra.run.smithi078.stdout: 5 102372 11.66 MB/sec execute 99 sec latency 80.726 ms 2024-09-21T12:29:02.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:02 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:29:02.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:02 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:29:02.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:02 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:29:02.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:02 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:29:02.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:02 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:29:02.271 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:02 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:29:02.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:02 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:29:02.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:02 smithi078 ceph-mon[23681]: Upgrade: Need to upgrade myself (mgr.smithi148.oyiqmx) 2024-09-21T12:29:02.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:02 smithi078 ceph-mon[23681]: Upgrade: Need to upgrade myself (mgr.smithi148.oyiqmx) 2024-09-21T12:29:02.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:02 smithi078 ceph-mon[23681]: Failing over to other MGR 2024-09-21T12:29:02.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:02 smithi078 ceph-mon[23681]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "mgr fail", "who": "smithi148.oyiqmx"}]: dispatch 2024-09-21T12:29:02.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:02 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "mgr fail", "who": "smithi148.oyiqmx"}]: dispatch 2024-09-21T12:29:02.272 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:02 smithi078 ceph-mon[23681]: osdmap e71: 8 total, 8 up, 8 in 2024-09-21T12:29:02.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:02 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:29:02.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:02 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:29:02.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:02 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:29:02.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:02 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:29:02.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:02 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:29:02.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:02 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' 2024-09-21T12:29:02.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:02 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:29:02.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:02 smithi148 ceph-mon[25818]: Upgrade: Need to upgrade myself (mgr.smithi148.oyiqmx) 2024-09-21T12:29:02.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:02 smithi148 ceph-mon[25818]: Upgrade: Need to upgrade myself (mgr.smithi148.oyiqmx) 2024-09-21T12:29:02.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:02 smithi148 ceph-mon[25818]: Failing over to other MGR 2024-09-21T12:29:02.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:02 smithi148 ceph-mon[25818]: from='mgr.24425 172.21.15.148:0/3118207884' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "mgr fail", "who": "smithi148.oyiqmx"}]: dispatch 2024-09-21T12:29:02.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:02 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "mgr fail", "who": "smithi148.oyiqmx"}]: dispatch 2024-09-21T12:29:02.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:02 smithi148 ceph-mon[25818]: osdmap e71: 8 total, 8 up, 8 in 2024-09-21T12:29:02.783 INFO:teuthology.orchestra.run.smithi078.stdout: 5 102758 11.72 MB/sec execute 100 sec latency 90.521 ms 2024-09-21T12:29:03.450 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:03 smithi078 ceph-mon[23681]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' cmd='[{"prefix": "mgr fail", "who": "smithi148.oyiqmx"}]': finished 2024-09-21T12:29:03.450 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:03 smithi078 ceph-mon[23681]: mgrmap e30: smithi078.ntwyan(active, starting, since 0.713123s) 2024-09-21T12:29:03.450 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:03 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-21T12:29:03.450 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:03 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi148"}]: dispatch 2024-09-21T12:29:03.451 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:03 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.ytpzwh"}]: dispatch 2024-09-21T12:29:03.451 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:03 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi148.uukezr"}]: dispatch 2024-09-21T12:29:03.451 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:03 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mgr metadata", "who": "smithi078.ntwyan", "id": "smithi078.ntwyan"}]: dispatch 2024-09-21T12:29:03.451 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:03 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T12:29:03.451 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:03 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T12:29:03.451 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:03 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T12:29:03.451 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:03 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T12:29:03.451 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:03 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T12:29:03.451 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:03 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T12:29:03.451 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:03 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T12:29:03.451 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:03 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T12:29:03.451 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:03 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T12:29:03.451 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:03 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T12:29:03.452 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:03 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T12:29:03.452 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:03 smithi078 ceph-mon[23681]: Manager daemon smithi078.ntwyan is now available 2024-09-21T12:29:03.452 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:03 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:29:03.452 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:03 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T12:29:03.452 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:03 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T12:29:03.452 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:03 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/mirror_snapshot_schedule"}]: dispatch 2024-09-21T12:29:03.452 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:03 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/trash_purge_schedule"}]: dispatch 2024-09-21T12:29:03.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:03 smithi148 ceph-mon[25818]: from='mgr.24425 ' entity='mgr.smithi148.oyiqmx' cmd='[{"prefix": "mgr fail", "who": "smithi148.oyiqmx"}]': finished 2024-09-21T12:29:03.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:03 smithi148 ceph-mon[25818]: mgrmap e30: smithi078.ntwyan(active, starting, since 0.713123s) 2024-09-21T12:29:03.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:03 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-21T12:29:03.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:03 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi148"}]: dispatch 2024-09-21T12:29:03.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:03 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.ytpzwh"}]: dispatch 2024-09-21T12:29:03.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:03 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi148.uukezr"}]: dispatch 2024-09-21T12:29:03.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:03 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mgr metadata", "who": "smithi078.ntwyan", "id": "smithi078.ntwyan"}]: dispatch 2024-09-21T12:29:03.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:03 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T12:29:03.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:03 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T12:29:03.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:03 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T12:29:03.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:03 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T12:29:03.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:03 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T12:29:03.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:03 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T12:29:03.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:03 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T12:29:03.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:03 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T12:29:03.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:03 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T12:29:03.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:03 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T12:29:03.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:03 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T12:29:03.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:03 smithi148 ceph-mon[25818]: Manager daemon smithi078.ntwyan is now available 2024-09-21T12:29:03.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:03 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:29:03.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:03 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T12:29:03.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:03 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T12:29:03.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:03 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/mirror_snapshot_schedule"}]: dispatch 2024-09-21T12:29:03.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:03 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/trash_purge_schedule"}]: dispatch 2024-09-21T12:29:03.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 103007 11.68 MB/sec execute 101 sec latency 49.282 ms 2024-09-21T12:29:04.495 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:04 smithi078 ceph-mon[23681]: mgrmap e31: smithi078.ntwyan(active, since 1.72474s) 2024-09-21T12:29:04.495 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:04 smithi078 ceph-mon[23681]: [21/Sep/2024:12:29:04] ENGINE Bus STARTING 2024-09-21T12:29:04.651 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:04 smithi148 ceph-mon[25818]: mgrmap e31: smithi078.ntwyan(active, since 1.72474s) 2024-09-21T12:29:04.652 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:04 smithi148 ceph-mon[25818]: [21/Sep/2024:12:29:04] ENGINE Bus STARTING 2024-09-21T12:29:04.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 103187 11.63 MB/sec execute 102 sec latency 102.419 ms 2024-09-21T12:29:05.519 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:05 smithi078 ceph-mon[23681]: [21/Sep/2024:12:29:04] ENGINE Serving on http://172.21.15.78:8765 2024-09-21T12:29:05.519 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:05 smithi078 ceph-mon[23681]: [21/Sep/2024:12:29:04] ENGINE Serving on https://172.21.15.78:7150 2024-09-21T12:29:05.519 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:05 smithi078 ceph-mon[23681]: [21/Sep/2024:12:29:04] ENGINE Bus STARTED 2024-09-21T12:29:05.519 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:05 smithi078 ceph-mon[23681]: [21/Sep/2024:12:29:04] ENGINE Error in HTTPServer.serve 2024-09-21T12:29:05.519 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Traceback (most recent call last): 2024-09-21T12:29:05.519 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T12:29:05.519 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self._connections.run(self.expiration_interval) 2024-09-21T12:29:05.519 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T12:29:05.520 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self._run(expiration_interval) 2024-09-21T12:29:05.520 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T12:29:05.520 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T12:29:05.520 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T12:29:05.520 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T12:29:05.520 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T12:29:05.520 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: s = self.context.wrap_socket( 2024-09-21T12:29:05.520 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T12:29:05.520 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: return self.sslsocket_class._create( 2024-09-21T12:29:05.520 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T12:29:05.520 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self.do_handshake() 2024-09-21T12:29:05.520 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T12:29:05.520 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self._sslobj.do_handshake() 2024-09-21T12:29:05.520 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T12:29:05.520 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:05 smithi078 ceph-mon[23681]: pgmap v4: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T12:29:05.738 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:05 smithi148 ceph-mon[25818]: [21/Sep/2024:12:29:04] ENGINE Serving on http://172.21.15.78:8765 2024-09-21T12:29:05.738 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:05 smithi148 ceph-mon[25818]: [21/Sep/2024:12:29:04] ENGINE Serving on https://172.21.15.78:7150 2024-09-21T12:29:05.738 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:05 smithi148 ceph-mon[25818]: [21/Sep/2024:12:29:04] ENGINE Bus STARTED 2024-09-21T12:29:05.738 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:05 smithi148 ceph-mon[25818]: [21/Sep/2024:12:29:04] ENGINE Error in HTTPServer.serve 2024-09-21T12:29:05.738 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Traceback (most recent call last): 2024-09-21T12:29:05.739 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T12:29:05.739 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: self._connections.run(self.expiration_interval) 2024-09-21T12:29:05.739 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T12:29:05.739 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: self._run(expiration_interval) 2024-09-21T12:29:05.739 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T12:29:05.739 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T12:29:05.739 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T12:29:05.739 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T12:29:05.739 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T12:29:05.739 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: s = self.context.wrap_socket( 2024-09-21T12:29:05.739 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T12:29:05.739 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: return self.sslsocket_class._create( 2024-09-21T12:29:05.739 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T12:29:05.739 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: self.do_handshake() 2024-09-21T12:29:05.739 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T12:29:05.740 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: self._sslobj.do_handshake() 2024-09-21T12:29:05.740 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T12:29:05.740 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:05 smithi148 ceph-mon[25818]: pgmap v4: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T12:29:05.783 INFO:teuthology.orchestra.run.smithi078.stdout: 5 103429 11.54 MB/sec execute 103 sec latency 70.694 ms 2024-09-21T12:29:06.551 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:06 smithi078 ceph-mon[23681]: mgrmap e32: smithi078.ntwyan(active, since 3s) 2024-09-21T12:29:06.731 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:06 smithi148 ceph-mon[25818]: mgrmap e32: smithi078.ntwyan(active, since 3s) 2024-09-21T12:29:06.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 103644 11.44 MB/sec execute 104 sec latency 80.969 ms 2024-09-21T12:29:06.947 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-21T12:29:07.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:07 smithi078 ceph-mon[23681]: pgmap v5: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T12:29:07.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:07 smithi078 ceph-mon[23681]: from='client.24539 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:29:07.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:07 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:07.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:07 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:07.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 103808 11.37 MB/sec execute 105 sec latency 99.471 ms 2024-09-21T12:29:07.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:07 smithi148 ceph-mon[25818]: pgmap v5: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T12:29:07.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:07 smithi148 ceph-mon[25818]: from='client.24539 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:29:07.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:07 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:07.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:07 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:08.045 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T12:29:08.045 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (80s) 14s ago 10m 18.5M - 0.27.0 11f11916f8cd e6e67ab07dec 2024-09-21T12:29:08.045 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (10m) 14s ago 10m 6400k - 17.2.0 e1d6a67b021e 4a209e5da5ce 2024-09-21T12:29:08.045 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi148 smithi148 running (9m) 0s ago 9m 6405k - 17.2.0 e1d6a67b021e 0fa5e612307f 2024-09-21T12:29:08.045 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (71s) 14s ago 10m 45.7M - dad864ee21e9 2a140c36091b 2024-09-21T12:29:08.046 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.ytpzwh smithi078 running (4m) 14s ago 4m 36.7M - 17.2.0 e1d6a67b021e 5e9b109502a2 2024-09-21T12:29:08.046 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi148.uukezr smithi148 running (4m) 0s ago 4m 12.9M - 17.2.0 e1d6a67b021e b7e8aa0f3a6c 2024-09-21T12:29:08.046 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.ntwyan smithi078 *:8443,9283,8765 running (25s) 14s ago 11m 84.9M - 19.3.0-5104-g44178494 b415816db8b4 a61baedebf43 2024-09-21T12:29:08.046 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi148.oyiqmx smithi148 *:8443,9283 running (2m) 0s ago 9m 126M - 19.3.0-5104-g44178494 b415816db8b4 58467ce9a26b 2024-09-21T12:29:08.046 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (11m) 14s ago 11m 50.8M 2048M 17.2.0 e1d6a67b021e 8261b0b2c7e0 2024-09-21T12:29:08.046 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi148 smithi148 running (9m) 0s ago 9m 44.0M 2048M 17.2.0 e1d6a67b021e fda961db230f 2024-09-21T12:29:08.046 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.ydbhru smithi078 *:2049 running (3m) 14s ago 3m 266M - 4.0 e1d6a67b021e 7e5b0274ac38 2024-09-21T12:29:08.046 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi148.einuzt smithi148 *:2049 running (3m) 0s ago 3m 82.0M - 4.0 e1d6a67b021e 76a8b0eabede 2024-09-21T12:29:08.046 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (63s) 14s ago 10m 8304k - 1.7.0 72c9c2088986 301adb4e9e22 2024-09-21T12:29:08.046 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi148 smithi148 *:9100 running (36s) 0s ago 9m 9.91M - 1.7.0 72c9c2088986 0cceff4bf342 2024-09-21T12:29:08.046 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (8m) 14s ago 8m 377M 1936M 17.2.0 e1d6a67b021e d4d5e9977070 2024-09-21T12:29:08.046 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (8m) 14s ago 8m 295M 1936M 17.2.0 e1d6a67b021e b9a4b952233b 2024-09-21T12:29:08.046 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (7m) 14s ago 7m 264M 1936M 17.2.0 e1d6a67b021e 829fb8c2d827 2024-09-21T12:29:08.047 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (7m) 14s ago 7m 358M 1936M 17.2.0 e1d6a67b021e ef942c8556d0 2024-09-21T12:29:08.047 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi148 running (7m) 0s ago 7m 365M 2704M 17.2.0 e1d6a67b021e 62eefdd572c0 2024-09-21T12:29:08.047 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi148 running (6m) 0s ago 6m 398M 2704M 17.2.0 e1d6a67b021e 7b44be78baca 2024-09-21T12:29:08.047 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi148 running (6m) 0s ago 6m 340M 2704M 17.2.0 e1d6a67b021e 3d23057910ea 2024-09-21T12:29:08.047 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi148 running (5m) 0s ago 5m 463M 2704M 17.2.0 e1d6a67b021e f9e4db9d35fa 2024-09-21T12:29:08.047 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (41s) 14s ago 9m 39.6M - 2.51.0 1d3b7f56885b bd590dad0913 2024-09-21T12:29:08.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:08 smithi148 ceph-mon[25818]: mgrmap e33: smithi078.ntwyan(active, since 5s) 2024-09-21T12:29:08.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:08 smithi148 ceph-mon[25818]: from='client.24543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:29:08.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:08 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:08.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:08 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:08.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:08 smithi148 ceph-mon[25818]: from='client.24547 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:29:08.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:08 smithi148 ceph-mon[25818]: from='mgr.? 172.21.15.148:0/378120576' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi148.oyiqmx/crt"}]: dispatch 2024-09-21T12:29:08.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:08 smithi148 ceph-mon[25818]: from='mgr.? 172.21.15.148:0/378120576' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T12:29:08.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:08 smithi148 ceph-mon[25818]: from='mgr.? 172.21.15.148:0/378120576' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi148.oyiqmx/key"}]: dispatch 2024-09-21T12:29:08.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:08 smithi148 ceph-mon[25818]: from='mgr.? 172.21.15.148:0/378120576' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T12:29:08.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:08 smithi148 ceph-mon[25818]: Standby manager daemon smithi148.oyiqmx started 2024-09-21T12:29:08.596 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:29:08.596 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-21T12:29:08.596 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:29:08.596 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:29:08.597 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-21T12:29:08.597 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 1 2024-09-21T12:29:08.597 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:29:08.597 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-21T12:29:08.597 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T12:29:08.597 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:29:08.597 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-21T12:29:08.597 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:29:08.597 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:29:08.597 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-21T12:29:08.597 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-21T12:29:08.597 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 1 2024-09-21T12:29:08.597 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T12:29:08.597 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:29:08.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:08 smithi078 ceph-mon[23681]: mgrmap e33: smithi078.ntwyan(active, since 5s) 2024-09-21T12:29:08.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:08 smithi078 ceph-mon[23681]: from='client.24543 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:29:08.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:08 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:08.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:08 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:08.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:08 smithi078 ceph-mon[23681]: from='client.24547 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:29:08.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:08 smithi078 ceph-mon[23681]: from='mgr.? 172.21.15.148:0/378120576' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi148.oyiqmx/crt"}]: dispatch 2024-09-21T12:29:08.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:08 smithi078 ceph-mon[23681]: from='mgr.? 172.21.15.148:0/378120576' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T12:29:08.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:08 smithi078 ceph-mon[23681]: from='mgr.? 172.21.15.148:0/378120576' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi148.oyiqmx/key"}]: dispatch 2024-09-21T12:29:08.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:08 smithi078 ceph-mon[23681]: from='mgr.? 172.21.15.148:0/378120576' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T12:29:08.626 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:08 smithi078 ceph-mon[23681]: Standby manager daemon smithi148.oyiqmx started 2024-09-21T12:29:08.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 104068 11.35 MB/sec execute 106 sec latency 171.322 ms 2024-09-21T12:29:09.162 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:29:09.162 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5", 2024-09-21T12:29:09.162 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-21T12:29:09.162 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T12:29:09.162 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-21T12:29:09.162 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr" 2024-09-21T12:29:09.162 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-21T12:29:09.162 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "2/23 daemons upgraded", 2024-09-21T12:29:09.162 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "", 2024-09-21T12:29:09.162 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-21T12:29:09.163 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:29:09.633 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:09 smithi078 ceph-mon[23681]: pgmap v6: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T12:29:09.633 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:09 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/3301399818' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T12:29:09.633 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:09 smithi078 ceph-mon[23681]: mgrmap e34: smithi078.ntwyan(active, since 7s), standbys: smithi148.oyiqmx 2024-09-21T12:29:09.633 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:09 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mgr metadata", "who": "smithi148.oyiqmx", "id": "smithi148.oyiqmx"}]: dispatch 2024-09-21T12:29:09.736 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-21T12:29:09.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 104475 11.31 MB/sec execute 107 sec latency 100.520 ms 2024-09-21T12:29:09.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:09 smithi148 ceph-mon[25818]: pgmap v6: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T12:29:09.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:09 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/3301399818' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T12:29:09.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:09 smithi148 ceph-mon[25818]: mgrmap e34: smithi078.ntwyan(active, since 7s), standbys: smithi148.oyiqmx 2024-09-21T12:29:09.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:09 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mgr metadata", "who": "smithi148.oyiqmx", "id": "smithi148.oyiqmx"}]: dispatch 2024-09-21T12:29:10.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:10 smithi078 ceph-mon[23681]: from='client.24553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:29:10.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:10 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/2257689121' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T12:29:10.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 104865 11.25 MB/sec execute 108 sec latency 77.576 ms 2024-09-21T12:29:10.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:10 smithi148 ceph-mon[25818]: from='client.24553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:29:10.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:10 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/2257689121' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T12:29:11.670 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:11 smithi078 ceph-mon[23681]: pgmap v7: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 32 KiB/s rd, 2.5 MiB/s wr, 264 op/s 2024-09-21T12:29:11.670 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:11 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:11.670 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:11 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:11.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 105277 11.23 MB/sec execute 109 sec latency 54.684 ms 2024-09-21T12:29:11.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:11 smithi148 ceph-mon[25818]: pgmap v7: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 32 KiB/s rd, 2.5 MiB/s wr, 264 op/s 2024-09-21T12:29:11.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:11 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:11.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:11 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:12.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 105800 11.32 MB/sec execute 110 sec latency 41.338 ms 2024-09-21T12:29:13.211 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:13 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:13.211 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:13 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:13.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:13 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:13.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:13 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:13.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 106339 11.42 MB/sec execute 111 sec latency 65.147 ms 2024-09-21T12:29:14.227 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:14 smithi078 ceph-mon[23681]: pgmap v8: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 2.0 MiB/s wr, 204 op/s 2024-09-21T12:29:14.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:14 smithi148 ceph-mon[25818]: pgmap v8: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 2.0 MiB/s wr, 204 op/s 2024-09-21T12:29:14.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 106639 11.41 MB/sec execute 112 sec latency 52.478 ms 2024-09-21T12:29:15.252 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:15 smithi078 ceph-mon[23681]: pgmap v9: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.4 MiB/s wr, 440 op/s 2024-09-21T12:29:15.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:15 smithi148 ceph-mon[25818]: pgmap v9: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.4 MiB/s wr, 440 op/s 2024-09-21T12:29:15.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 106885 11.35 MB/sec execute 113 sec latency 57.676 ms 2024-09-21T12:29:16.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 107124 11.27 MB/sec execute 114 sec latency 169.276 ms 2024-09-21T12:29:17.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:17 smithi078 ceph-mon[23681]: pgmap v10: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 4.0 MiB/s wr, 399 op/s 2024-09-21T12:29:17.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 107347 11.22 MB/sec execute 115 sec latency 80.310 ms 2024-09-21T12:29:17.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:17 smithi148 ceph-mon[25818]: pgmap v10: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 4.0 MiB/s wr, 399 op/s 2024-09-21T12:29:18.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:18 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T12:29:18.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 107683 11.22 MB/sec execute 116 sec latency 79.881 ms 2024-09-21T12:29:18.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:18 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T12:29:19.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 107967 11.16 MB/sec execute 117 sec latency 88.877 ms 2024-09-21T12:29:20.153 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:20 smithi078 ceph-mon[23681]: pgmap v11: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 4.0 MiB/s wr, 399 op/s 2024-09-21T12:29:20.153 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:20 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:20.153 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:20 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:20.153 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:20 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:29:20.153 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:20 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:29:20.153 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:20 smithi078 ceph-mon[23681]: Updating smithi078:/etc/ceph/ceph.conf 2024-09-21T12:29:20.153 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:20 smithi078 ceph-mon[23681]: Updating smithi148:/etc/ceph/ceph.conf 2024-09-21T12:29:20.153 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:20 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:20.153 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:20 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:20.153 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:20 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:20.153 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:20 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:20.154 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:20 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:20.154 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:20 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:20.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:20 smithi148 ceph-mon[25818]: pgmap v11: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 4.0 MiB/s wr, 399 op/s 2024-09-21T12:29:20.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:20 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:20.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:20 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:20.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:20 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:29:20.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:20 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:29:20.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:20 smithi148 ceph-mon[25818]: Updating smithi078:/etc/ceph/ceph.conf 2024-09-21T12:29:20.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:20 smithi148 ceph-mon[25818]: Updating smithi148:/etc/ceph/ceph.conf 2024-09-21T12:29:20.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:20 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:20.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:20 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:20.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:20 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:20.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:20 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:20.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:20 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:20.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:20 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:20.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 108440 11.10 MB/sec execute 118 sec latency 95.498 ms 2024-09-21T12:29:21.167 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:21 smithi078 ceph-mon[23681]: Updating smithi078:/var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T12:29:21.168 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:21 smithi078 ceph-mon[23681]: Updating smithi148:/var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T12:29:21.168 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:21 smithi078 ceph-mon[23681]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-09-21T12:29:21.168 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:21 smithi078 ceph-mon[23681]: Updating smithi148:/etc/ceph/ceph.client.admin.keyring 2024-09-21T12:29:21.168 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:21 smithi078 ceph-mon[23681]: Updating smithi078:/var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T12:29:21.168 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:21 smithi078 ceph-mon[23681]: Updating smithi148:/var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T12:29:21.168 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:21 smithi078 ceph-mon[23681]: Reconfiguring prometheus.smithi078 (dependencies changed)... 2024-09-21T12:29:21.168 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:21 smithi078 ceph-mon[23681]: Reconfiguring daemon prometheus.smithi078 on smithi078 2024-09-21T12:29:21.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:21 smithi148 ceph-mon[25818]: Updating smithi078:/var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T12:29:21.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:21 smithi148 ceph-mon[25818]: Updating smithi148:/var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T12:29:21.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:21 smithi148 ceph-mon[25818]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-09-21T12:29:21.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:21 smithi148 ceph-mon[25818]: Updating smithi148:/etc/ceph/ceph.client.admin.keyring 2024-09-21T12:29:21.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:21 smithi148 ceph-mon[25818]: Updating smithi078:/var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T12:29:21.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:21 smithi148 ceph-mon[25818]: Updating smithi148:/var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T12:29:21.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:21 smithi148 ceph-mon[25818]: Reconfiguring prometheus.smithi078 (dependencies changed)... 2024-09-21T12:29:21.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:21 smithi148 ceph-mon[25818]: Reconfiguring daemon prometheus.smithi078 on smithi078 2024-09-21T12:29:21.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 108760 11.08 MB/sec execute 119 sec latency 87.891 ms 2024-09-21T12:29:22.229 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:22 smithi078 ceph-mon[23681]: pgmap v12: 97 pgs: 97 active+clean; 338 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 5.9 MiB/s wr, 583 op/s 2024-09-21T12:29:22.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:22 smithi148 ceph-mon[25818]: pgmap v12: 97 pgs: 97 active+clean; 338 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 5.9 MiB/s wr, 583 op/s 2024-09-21T12:29:22.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 109027 11.05 MB/sec execute 120 sec latency 90.668 ms 2024-09-21T12:29:23.234 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:23 smithi078 ceph-mon[23681]: pgmap v13: 97 pgs: 97 active+clean; 338 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.5 MiB/s wr, 432 op/s 2024-09-21T12:29:23.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:23 smithi148 ceph-mon[25818]: pgmap v13: 97 pgs: 97 active+clean; 338 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.5 MiB/s wr, 432 op/s 2024-09-21T12:29:23.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 109399 11.10 MB/sec execute 121 sec latency 56.556 ms 2024-09-21T12:29:24.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 109782 11.12 MB/sec execute 122 sec latency 75.476 ms 2024-09-21T12:29:25.535 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:25 smithi078 ceph-mon[23681]: pgmap v14: 97 pgs: 97 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.2 MiB/s wr, 636 op/s 2024-09-21T12:29:25.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 110086 11.14 MB/sec execute 123 sec latency 62.266 ms 2024-09-21T12:29:25.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:25 smithi148 ceph-mon[25818]: pgmap v14: 97 pgs: 97 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.2 MiB/s wr, 636 op/s 2024-09-21T12:29:26.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 110352 11.11 MB/sec execute 124 sec latency 64.839 ms 2024-09-21T12:29:27.568 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:27 smithi078 ceph-mon[23681]: pgmap v15: 97 pgs: 97 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.7 MiB/s wr, 388 op/s 2024-09-21T12:29:27.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 110589 11.04 MB/sec execute 125 sec latency 76.806 ms 2024-09-21T12:29:27.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:27 smithi148 ceph-mon[25818]: pgmap v15: 97 pgs: 97 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.7 MiB/s wr, 388 op/s 2024-09-21T12:29:28.784 INFO:teuthology.orchestra.run.smithi078.stdout: 5 110856 10.98 MB/sec execute 126 sec latency 64.349 ms 2024-09-21T12:29:29.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:29 smithi078 ceph-mon[23681]: pgmap v16: 97 pgs: 97 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.7 MiB/s wr, 388 op/s 2024-09-21T12:29:29.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:29 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:29.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:29 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:29.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:29 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T12:29:29.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:29 smithi078 ceph-mon[23681]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T12:29:29.596 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:29 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:29:29.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 111200 10.96 MB/sec execute 127 sec latency 47.729 ms 2024-09-21T12:29:29.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:29 smithi148 ceph-mon[25818]: pgmap v16: 97 pgs: 97 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.7 MiB/s wr, 388 op/s 2024-09-21T12:29:29.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:29 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:29.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:29 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:29.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:29 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T12:29:29.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:29 smithi148 ceph-mon[25818]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T12:29:29.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:29 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:29:30.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 111641 10.95 MB/sec execute 128 sec latency 48.216 ms 2024-09-21T12:29:31.589 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:31 smithi078 ceph-mon[23681]: pgmap v17: 97 pgs: 97 active+clean; 342 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.8 MiB/s wr, 563 op/s 2024-09-21T12:29:31.589 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:31 smithi078 ceph-mon[23681]: Upgrade: Updating mgr.smithi148.oyiqmx 2024-09-21T12:29:31.589 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:31 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi148.oyiqmx", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T12:29:31.589 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:31 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T12:29:31.589 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:31 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:29:31.589 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:31 smithi078 ceph-mon[23681]: Deploying daemon mgr.smithi148.oyiqmx on smithi148 2024-09-21T12:29:31.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 112232 10.96 MB/sec execute 129 sec latency 42.324 ms 2024-09-21T12:29:31.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:31 smithi148 ceph-mon[25818]: pgmap v17: 97 pgs: 97 active+clean; 342 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.8 MiB/s wr, 563 op/s 2024-09-21T12:29:31.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:31 smithi148 ceph-mon[25818]: Upgrade: Updating mgr.smithi148.oyiqmx 2024-09-21T12:29:31.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:31 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi148.oyiqmx", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T12:29:31.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:31 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T12:29:31.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:31 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:29:31.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:31 smithi148 ceph-mon[25818]: Deploying daemon mgr.smithi148.oyiqmx on smithi148 2024-09-21T12:29:32.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 112837 11.03 MB/sec execute 130 sec latency 32.490 ms 2024-09-21T12:29:33.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:33 smithi078 ceph-mon[23681]: pgmap v18: 97 pgs: 97 active+clean; 342 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.9 MiB/s wr, 379 op/s 2024-09-21T12:29:33.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:33 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T12:29:33.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:33 smithi148 ceph-mon[25818]: pgmap v18: 97 pgs: 97 active+clean; 342 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.9 MiB/s wr, 379 op/s 2024-09-21T12:29:33.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:33 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T12:29:33.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 113124 11.02 MB/sec execute 131 sec latency 69.663 ms 2024-09-21T12:29:34.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 113424 11.07 MB/sec execute 132 sec latency 56.708 ms 2024-09-21T12:29:35.450 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:35 smithi078 ceph-mon[23681]: pgmap v19: 97 pgs: 97 active+clean; 340 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.4 MiB/s wr, 645 op/s 2024-09-21T12:29:35.450 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:35 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:35.450 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:35 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:35.450 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:35 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:29:35.450 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:35 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:35.450 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:35 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:35.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 113691 11.04 MB/sec execute 133 sec latency 126.617 ms 2024-09-21T12:29:35.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:35 smithi148 ceph-mon[25818]: pgmap v19: 97 pgs: 97 active+clean; 340 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.4 MiB/s wr, 645 op/s 2024-09-21T12:29:35.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:35 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:35.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:35 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:35.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:35 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:29:35.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:35 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:35.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:35 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:36.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 113944 11.00 MB/sec execute 134 sec latency 66.525 ms 2024-09-21T12:29:37.523 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:37 smithi078 ceph-mon[23681]: pgmap v20: 97 pgs: 97 active+clean; 340 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.6 MiB/s wr, 441 op/s 2024-09-21T12:29:37.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:37 smithi148 ceph-mon[25818]: pgmap v20: 97 pgs: 97 active+clean; 340 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.6 MiB/s wr, 441 op/s 2024-09-21T12:29:37.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 114260 10.94 MB/sec execute 135 sec latency 77.132 ms 2024-09-21T12:29:38.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 114561 10.91 MB/sec execute 136 sec latency 86.940 ms 2024-09-21T12:29:39.553 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:39 smithi078 ceph-mon[23681]: pgmap v21: 97 pgs: 97 active+clean; 340 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.6 MiB/s wr, 441 op/s 2024-09-21T12:29:39.554 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:39 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:39.554 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:39 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:39.554 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:39 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:39.554 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:39 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:39.725 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:39 smithi148 ceph-mon[25818]: pgmap v21: 97 pgs: 97 active+clean; 340 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.6 MiB/s wr, 441 op/s 2024-09-21T12:29:39.725 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:39 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:39.726 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:39 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:39.726 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:39 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:39.726 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:39 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:39.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 114902 10.90 MB/sec execute 137 sec latency 105.958 ms 2024-09-21T12:29:40.239 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-21T12:29:40.669 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:40 smithi078 ceph-mon[23681]: Standby manager daemon smithi148.oyiqmx restarted 2024-09-21T12:29:40.670 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:40 smithi078 ceph-mon[23681]: Standby manager daemon smithi148.oyiqmx started 2024-09-21T12:29:40.670 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:40 smithi078 ceph-mon[23681]: from='mgr.? 172.21.15.148:0/1040175479' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi148.oyiqmx/crt"}]: dispatch 2024-09-21T12:29:40.670 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:40 smithi078 ceph-mon[23681]: from='mgr.? 172.21.15.148:0/1040175479' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T12:29:40.670 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:40 smithi078 ceph-mon[23681]: from='mgr.? 172.21.15.148:0/1040175479' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi148.oyiqmx/key"}]: dispatch 2024-09-21T12:29:40.670 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:40 smithi078 ceph-mon[23681]: from='mgr.? 172.21.15.148:0/1040175479' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T12:29:40.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 115232 10.86 MB/sec execute 138 sec latency 113.531 ms 2024-09-21T12:29:40.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:40 smithi148 ceph-mon[25818]: Standby manager daemon smithi148.oyiqmx restarted 2024-09-21T12:29:40.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:40 smithi148 ceph-mon[25818]: Standby manager daemon smithi148.oyiqmx started 2024-09-21T12:29:40.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:40 smithi148 ceph-mon[25818]: from='mgr.? 172.21.15.148:0/1040175479' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi148.oyiqmx/crt"}]: dispatch 2024-09-21T12:29:40.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:40 smithi148 ceph-mon[25818]: from='mgr.? 172.21.15.148:0/1040175479' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T12:29:40.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:40 smithi148 ceph-mon[25818]: from='mgr.? 172.21.15.148:0/1040175479' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi148.oyiqmx/key"}]: dispatch 2024-09-21T12:29:40.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:40 smithi148 ceph-mon[25818]: from='mgr.? 172.21.15.148:0/1040175479' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T12:29:41.324 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T12:29:41.324 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (114s) 29s ago 10m 18.5M - 0.27.0 11f11916f8cd e6e67ab07dec 2024-09-21T12:29:41.324 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (10m) 29s ago 10m 6400k - 17.2.0 e1d6a67b021e 4a209e5da5ce 2024-09-21T12:29:41.324 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi148 smithi148 running (10m) 2s ago 10m 6405k - 17.2.0 e1d6a67b021e 0fa5e612307f 2024-09-21T12:29:41.324 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (104s) 29s ago 10m 46.7M - dad864ee21e9 2a140c36091b 2024-09-21T12:29:41.324 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.ytpzwh smithi078 running (4m) 29s ago 4m 36.9M - 17.2.0 e1d6a67b021e 5e9b109502a2 2024-09-21T12:29:41.324 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi148.uukezr smithi148 running (4m) 2s ago 4m 13.1M - 17.2.0 e1d6a67b021e b7e8aa0f3a6c 2024-09-21T12:29:41.324 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.ntwyan smithi078 *:8443,9283,8765 running (58s) 29s ago 12m 488M - 19.3.0-5104-g44178494 b415816db8b4 a61baedebf43 2024-09-21T12:29:41.325 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi148.oyiqmx smithi148 *:8443,9283,8765 running (7s) 2s ago 10m 89.5M - 19.3.0-5104-g44178494 b415816db8b4 0255535c6fdb 2024-09-21T12:29:41.325 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (12m) 29s ago 12m 56.5M 2048M 17.2.0 e1d6a67b021e 8261b0b2c7e0 2024-09-21T12:29:41.325 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi148 smithi148 running (10m) 2s ago 10m 44.0M 2048M 17.2.0 e1d6a67b021e fda961db230f 2024-09-21T12:29:41.325 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.ydbhru smithi078 *:2049 running (4m) 29s ago 4m 265M - 4.0 e1d6a67b021e 7e5b0274ac38 2024-09-21T12:29:41.325 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi148.einuzt smithi148 *:2049 running (4m) 2s ago 4m 82.1M - 4.0 e1d6a67b021e 76a8b0eabede 2024-09-21T12:29:41.325 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (96s) 29s ago 10m 9.94M - 1.7.0 72c9c2088986 301adb4e9e22 2024-09-21T12:29:41.325 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi148 smithi148 *:9100 running (69s) 2s ago 9m 9.78M - 1.7.0 72c9c2088986 0cceff4bf342 2024-09-21T12:29:41.325 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (9m) 29s ago 9m 405M 1936M 17.2.0 e1d6a67b021e d4d5e9977070 2024-09-21T12:29:41.325 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (8m) 29s ago 8m 321M 1936M 17.2.0 e1d6a67b021e b9a4b952233b 2024-09-21T12:29:41.325 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (8m) 29s ago 8m 268M 1936M 17.2.0 e1d6a67b021e 829fb8c2d827 2024-09-21T12:29:41.325 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (7m) 29s ago 7m 387M 1936M 17.2.0 e1d6a67b021e ef942c8556d0 2024-09-21T12:29:41.326 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi148 running (7m) 2s ago 7m 387M 2704M 17.2.0 e1d6a67b021e 62eefdd572c0 2024-09-21T12:29:41.326 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi148 running (7m) 2s ago 7m 440M 2704M 17.2.0 e1d6a67b021e 7b44be78baca 2024-09-21T12:29:41.326 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi148 running (6m) 2s ago 6m 380M 2704M 17.2.0 e1d6a67b021e 3d23057910ea 2024-09-21T12:29:41.326 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi148 running (6m) 2s ago 6m 491M 2704M 17.2.0 e1d6a67b021e f9e4db9d35fa 2024-09-21T12:29:41.326 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 starting - - - - 2024-09-21T12:29:41.697 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:41 smithi078 ceph-mon[23681]: from='client.14840 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:29:41.697 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:41 smithi078 ceph-mon[23681]: pgmap v22: 97 pgs: 97 active+clean; 344 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.7 MiB/s wr, 626 op/s 2024-09-21T12:29:41.697 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:41 smithi078 ceph-mon[23681]: mgrmap e35: smithi078.ntwyan(active, since 38s), standbys: smithi148.oyiqmx 2024-09-21T12:29:41.697 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:41 smithi078 ceph-mon[23681]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:29:41.788 INFO:teuthology.orchestra.run.smithi078.stdout: 5 115590 10.80 MB/sec execute 139 sec latency 55.370 ms 2024-09-21T12:29:41.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:41 smithi148 ceph-mon[25818]: from='client.14840 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:29:41.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:41 smithi148 ceph-mon[25818]: pgmap v22: 97 pgs: 97 active+clean; 344 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.7 MiB/s wr, 626 op/s 2024-09-21T12:29:41.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:41 smithi148 ceph-mon[25818]: mgrmap e35: smithi078.ntwyan(active, since 38s), standbys: smithi148.oyiqmx 2024-09-21T12:29:41.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:41 smithi148 ceph-mon[25818]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:29:41.911 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:29:41.911 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-21T12:29:41.911 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:29:41.911 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:29:41.911 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-21T12:29:41.911 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:29:41.911 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:29:41.911 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-21T12:29:41.911 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T12:29:41.912 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:29:41.912 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-21T12:29:41.912 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:29:41.912 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:29:41.912 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-21T12:29:41.912 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-21T12:29:41.912 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:29:41.912 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T12:29:41.912 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:29:42.426 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:29:42.426 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5", 2024-09-21T12:29:42.426 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-21T12:29:42.426 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T12:29:42.426 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-21T12:29:42.426 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr" 2024-09-21T12:29:42.426 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-21T12:29:42.426 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "2/23 daemons upgraded", 2024-09-21T12:29:42.426 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading mgr daemons", 2024-09-21T12:29:42.426 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-21T12:29:42.426 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:29:42.705 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:42 smithi078 ceph-mon[23681]: from='client.14848 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:29:42.705 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:42 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/3842057890' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T12:29:42.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 115967 10.82 MB/sec execute 140 sec latency 90.826 ms 2024-09-21T12:29:42.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:42 smithi148 ceph-mon[25818]: from='client.14848 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:29:42.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:42 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/3842057890' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T12:29:43.018 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-21T12:29:43.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 116257 10.80 MB/sec execute 141 sec latency 69.986 ms 2024-09-21T12:29:44.028 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:43 smithi078 ceph-mon[23681]: pgmap v23: 97 pgs: 97 active+clean; 344 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 451 op/s 2024-09-21T12:29:44.028 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:43 smithi078 ceph-mon[23681]: from='client.14856 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:29:44.028 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:43 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:44.028 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:43 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:44.028 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:43 smithi078 ceph-mon[23681]: from='client.? 172.21.15.78:0/1533645456' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T12:29:44.028 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:43 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:44.028 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:43 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:44.028 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:43 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:29:44.029 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:43 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:29:44.029 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:43 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:44.029 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:43 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:44.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:43 smithi148 ceph-mon[25818]: pgmap v23: 97 pgs: 97 active+clean; 344 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 451 op/s 2024-09-21T12:29:44.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:43 smithi148 ceph-mon[25818]: from='client.14856 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:29:44.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:43 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:44.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:43 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:44.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:43 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/1533645456' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-21T12:29:44.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:43 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:44.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:43 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:44.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:43 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:29:44.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:43 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T12:29:44.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:43 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:44.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:43 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:44.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 116616 10.84 MB/sec execute 142 sec latency 51.505 ms 2024-09-21T12:29:45.027 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:44 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:29:45.027 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:44 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T12:29:45.027 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:44 smithi078 ceph-mon[23681]: Upgrade: Setting container_image for all mgr 2024-09-21T12:29:45.027 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:44 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:45.027 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:44 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi078.ntwyan"}]: dispatch 2024-09-21T12:29:45.027 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:44 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi078.ntwyan"}]': finished 2024-09-21T12:29:45.027 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:44 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi148.oyiqmx"}]: dispatch 2024-09-21T12:29:45.027 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:44 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi148.oyiqmx"}]': finished 2024-09-21T12:29:45.028 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:44 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T12:29:45.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:44 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T12:29:45.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:44 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T12:29:45.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:44 smithi148 ceph-mon[25818]: Upgrade: Setting container_image for all mgr 2024-09-21T12:29:45.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:44 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:45.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:44 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi078.ntwyan"}]: dispatch 2024-09-21T12:29:45.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:44 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi078.ntwyan"}]': finished 2024-09-21T12:29:45.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:44 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi148.oyiqmx"}]: dispatch 2024-09-21T12:29:45.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:44 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi148.oyiqmx"}]': finished 2024-09-21T12:29:45.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:44 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T12:29:45.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 116947 10.86 MB/sec execute 143 sec latency 60.883 ms 2024-09-21T12:29:46.052 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:46 smithi078 ceph-mon[23681]: pgmap v24: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.3 MiB/s wr, 657 op/s 2024-09-21T12:29:46.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:46 smithi148 ceph-mon[25818]: pgmap v24: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.3 MiB/s wr, 657 op/s 2024-09-21T12:29:46.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 117240 10.86 MB/sec execute 144 sec latency 54.615 ms 2024-09-21T12:29:47.318 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:47 smithi078 ceph-mon[23681]: Upgrade: Updating mon.smithi078 2024-09-21T12:29:47.318 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:47 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:47.318 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:47 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T12:29:47.318 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:47 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T12:29:47.318 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:47 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:29:47.318 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:47 smithi078 ceph-mon[23681]: Deploying daemon mon.smithi078 on smithi078 2024-09-21T12:29:47.318 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:47 smithi078 ceph-mon[23681]: pgmap v25: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.8 MiB/s wr, 391 op/s 2024-09-21T12:29:47.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:47 smithi148 ceph-mon[25818]: Upgrade: Updating mon.smithi078 2024-09-21T12:29:47.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:47 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' 2024-09-21T12:29:47.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:47 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T12:29:47.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:47 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T12:29:47.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:47 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T12:29:47.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:47 smithi148 ceph-mon[25818]: Deploying daemon mon.smithi078 on smithi078 2024-09-21T12:29:47.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:47 smithi148 ceph-mon[25818]: pgmap v25: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.8 MiB/s wr, 391 op/s 2024-09-21T12:29:47.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 117468 10.82 MB/sec execute 145 sec latency 73.804 ms 2024-09-21T12:29:48.326 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:48 smithi078 ceph-mon[23681]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T12:29:48.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:48 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T12:29:48.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 117779 10.77 MB/sec execute 146 sec latency 72.363 ms 2024-09-21T12:29:49.086 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:48 smithi078 systemd[1]: Stopping Ceph mon.smithi078 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:29:49.337 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:49 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi078[23677]: 2024-09-21T12:29:49.254+0000 7f976526e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi078 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T12:29:49.337 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:49 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi078[23677]: 2024-09-21T12:29:49.254+0000 7f976526e700 -1 mon.smithi078@0(leader) e2 *** Got Signal Terminated *** 2024-09-21T12:29:49.337 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:49 smithi078 podman[63694]: 2024-09-21 12:29:49.275732893 +0000 UTC m=+0.223369156 container died 8261b0b2c7e041643c0685bc38c800cc93b33e816d6f9c3f3bdf60dc8d0b8830 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi078, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, com.redhat.component=centos-stream-container, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., architecture=x86_64, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD) 2024-09-21T12:29:49.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 118065 10.74 MB/sec execute 147 sec latency 37.545 ms 2024-09-21T12:29:50.120 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:50 smithi078 podman[63694]: 2024-09-21 12:29:50.070882305 +0000 UTC m=+1.018518551 container cleanup 8261b0b2c7e041643c0685bc38c800cc93b33e816d6f9c3f3bdf60dc8d0b8830 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi078, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, ceph=True, io.openshift.expose-services=, io.buildah.version=1.19.8, name=centos-stream, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, io.k8s.display-name=CentOS Stream 8) 2024-09-21T12:29:50.120 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:50 smithi078 bash[63694]: ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi078 2024-09-21T12:29:50.403 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:50 smithi078 podman[63707]: 2024-09-21 12:29:50.364570575 +0000 UTC m=+1.082226301 container remove 8261b0b2c7e041643c0685bc38c800cc93b33e816d6f9c3f3bdf60dc8d0b8830 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi078, GIT_CLEAN=True, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, GIT_BRANCH=HEAD, architecture=x86_64, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, name=centos-stream, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, ceph=True, io.buildah.version=1.19.8) 2024-09-21T12:29:50.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 118363 10.69 MB/sec execute 148 sec latency 86.921 ms 2024-09-21T12:29:50.915 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:50 smithi078 systemd[1]: ceph-6263deca-7813-11ef-baf6-efdc52797490@mon.smithi078.service: Deactivated successfully. 2024-09-21T12:29:50.915 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:50 smithi078 systemd[1]: Stopped Ceph mon.smithi078 for 6263deca-7813-11ef-baf6-efdc52797490. 2024-09-21T12:29:50.915 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:50 smithi078 systemd[1]: ceph-6263deca-7813-11ef-baf6-efdc52797490@mon.smithi078.service: Consumed 15.426s CPU time. 2024-09-21T12:29:51.167 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 systemd[1]: Starting Ceph mon.smithi078 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:29:51.419 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 podman[63848]: 2024-09-21 12:29:51.154222862 +0000 UTC m=+0.025533022 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:29:51.419 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 podman[63848]: 2024-09-21 12:29:51.351235701 +0000 UTC m=+0.222545847 container create 22b3097efbd8c02c4212806c49ca77f908da21e3a296e763947f8da487f3b6db (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi078, ceph=True, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T12:29:51.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 118739 10.70 MB/sec execute 149 sec latency 87.064 ms 2024-09-21T12:29:51.923 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 podman[63848]: 2024-09-21 12:29:51.834599433 +0000 UTC m=+0.705909599 container init 22b3097efbd8c02c4212806c49ca77f908da21e3a296e763947f8da487f3b6db (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi078, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0842-4417849, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-21T12:29:51.923 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 podman[63848]: 2024-09-21 12:29:51.844096214 +0000 UTC m=+0.715406372 container start 22b3097efbd8c02c4212806c49ca77f908da21e3a296e763947f8da487f3b6db (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi078, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0842-4417849, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-21T12:29:51.923 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T12:29:51.924 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev), process ceph-mon, pid 2 2024-09-21T12:29:51.924 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: pidfile_write: ignore empty --pid-file 2024-09-21T12:29:51.924 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: load: jerasure load: lrc 2024-09-21T12:29:51.924 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: RocksDB version: 7.9.2 2024-09-21T12:29:51.924 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Git sha 0 2024-09-21T12:29:51.924 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Compile date 2024-09-21 09:24:23 2024-09-21T12:29:51.924 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: DB SUMMARY 2024-09-21T12:29:51.924 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: DB Session ID: KQZ5A6VQPJEZE1O8I7XS 2024-09-21T12:29:51.924 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: CURRENT file: CURRENT 2024-09-21T12:29:51.924 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T12:29:51.924 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: MANIFEST file: MANIFEST-000015 size: 979 Bytes 2024-09-21T12:29:51.924 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi078/store.db dir, Total Num: 1, files: 000030.sst 2024-09-21T12:29:51.924 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi078/store.db: 000028.log size: 2334992 ; 2024-09-21T12:29:51.925 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.error_if_exists: 0 2024-09-21T12:29:51.925 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.create_if_missing: 0 2024-09-21T12:29:51.925 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.paranoid_checks: 1 2024-09-21T12:29:51.925 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-21T12:29:51.925 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T12:29:51.925 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-21T12:29:51.925 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.env: 0x55d5bfb20120 2024-09-21T12:29:51.925 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.fs: PosixFileSystem 2024-09-21T12:29:51.925 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.info_log: 0x55d5c0a178e0 2024-09-21T12:29:51.925 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T12:29:51.926 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.statistics: (nil) 2024-09-21T12:29:51.926 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.use_fsync: 0 2024-09-21T12:29:51.927 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.max_log_file_size: 0 2024-09-21T12:29:51.927 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T12:29:51.927 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T12:29:51.927 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T12:29:51.927 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T12:29:51.927 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.allow_fallocate: 1 2024-09-21T12:29:51.927 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T12:29:51.927 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T12:29:51.927 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.use_direct_reads: 0 2024-09-21T12:29:51.927 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T12:29:51.927 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T12:29:51.927 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.db_log_dir: 2024-09-21T12:29:51.927 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.wal_dir: 2024-09-21T12:29:51.927 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T12:29:51.928 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T12:29:51.928 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T12:29:51.928 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T12:29:51.928 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T12:29:51.928 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T12:29:51.928 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T12:29:51.928 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T12:29:51.928 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.write_buffer_manager: 0x55d5c0a1bb80 2024-09-21T12:29:51.928 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T12:29:51.928 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T12:29:51.928 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T12:29:51.928 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T12:29:51.928 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T12:29:51.928 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T12:29:51.929 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T12:29:51.929 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T12:29:51.929 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.unordered_write: 0 2024-09-21T12:29:51.929 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T12:29:51.929 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T12:29:51.929 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T12:29:51.929 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T12:29:51.929 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.row_cache: None 2024-09-21T12:29:51.929 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.wal_filter: None 2024-09-21T12:29:51.929 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T12:29:51.929 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T12:29:51.929 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.two_write_queues: 0 2024-09-21T12:29:51.930 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T12:29:51.930 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.wal_compression: 0 2024-09-21T12:29:51.930 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.atomic_flush: 0 2024-09-21T12:29:51.930 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T12:29:51.930 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T12:29:51.930 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T12:29:51.930 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.log_readahead_size: 0 2024-09-21T12:29:51.930 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T12:29:51.930 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T12:29:51.930 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T12:29:51.930 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T12:29:51.930 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T12:29:51.930 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T12:29:51.931 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-21T12:29:51.931 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.max_background_jobs: 2 2024-09-21T12:29:51.931 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.max_background_compactions: -1 2024-09-21T12:29:51.931 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.max_subcompactions: 1 2024-09-21T12:29:51.931 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T12:29:51.931 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T12:29:51.931 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T12:29:51.931 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T12:29:51.931 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T12:29:51.931 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T12:29:51.931 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T12:29:51.931 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T12:29:51.932 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.max_open_files: -1 2024-09-21T12:29:51.932 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T12:29:51.932 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T12:29:51.932 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T12:29:51.932 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T12:29:51.932 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.max_background_flushes: -1 2024-09-21T12:29:51.932 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Compression algorithms supported: 2024-09-21T12:29:51.932 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: kZSTD supported: 0 2024-09-21T12:29:51.932 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: kXpressCompression supported: 0 2024-09-21T12:29:51.932 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: kBZip2Compression supported: 0 2024-09-21T12:29:51.932 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T12:29:51.932 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: kLZ4Compression supported: 1 2024-09-21T12:29:51.932 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: kZlibCompression supported: 1 2024-09-21T12:29:51.933 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T12:29:51.933 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: kSnappyCompression supported: 1 2024-09-21T12:29:51.933 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T12:29:51.933 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-21T12:29:51.933 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi078/store.db/MANIFEST-000015 2024-09-21T12:29:51.933 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-21T12:29:51.933 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T12:29:51.933 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.merge_operator: 2024-09-21T12:29:51.933 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.compaction_filter: None 2024-09-21T12:29:51.933 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T12:29:51.933 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T12:29:51.933 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T12:29:51.933 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T12:29:51.934 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55d5c0a17540) 2024-09-21T12:29:51.934 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: cache_index_and_filter_blocks: 1 2024-09-21T12:29:51.934 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T12:29:51.934 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T12:29:51.934 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: pin_top_level_index_and_filter: 1 2024-09-21T12:29:51.934 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: index_type: 0 2024-09-21T12:29:51.934 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: data_block_index_type: 0 2024-09-21T12:29:51.934 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: index_shortening: 1 2024-09-21T12:29:51.934 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T12:29:51.934 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: checksum: 4 2024-09-21T12:29:51.934 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: no_block_cache: 0 2024-09-21T12:29:51.934 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache: 0x55d5c0a0b610 2024-09-21T12:29:51.934 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache_name: BinnedLRUCache 2024-09-21T12:29:51.935 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache_options: 2024-09-21T12:29:51.935 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: capacity : 536870912 2024-09-21T12:29:51.935 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: num_shard_bits : 4 2024-09-21T12:29:51.935 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: strict_capacity_limit : 0 2024-09-21T12:29:51.935 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: high_pri_pool_ratio: 0.000 2024-09-21T12:29:51.935 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache_compressed: (nil) 2024-09-21T12:29:51.935 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: persistent_cache: (nil) 2024-09-21T12:29:51.935 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_size: 4096 2024-09-21T12:29:51.935 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_size_deviation: 10 2024-09-21T12:29:51.935 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_restart_interval: 16 2024-09-21T12:29:51.935 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: index_block_restart_interval: 1 2024-09-21T12:29:51.935 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: metadata_block_size: 4096 2024-09-21T12:29:51.935 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: partition_filters: 0 2024-09-21T12:29:51.936 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: use_delta_encoding: 1 2024-09-21T12:29:51.936 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: filter_policy: bloomfilter 2024-09-21T12:29:51.936 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: whole_key_filtering: 1 2024-09-21T12:29:51.936 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: verify_compression: 0 2024-09-21T12:29:51.936 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: read_amp_bytes_per_bit: 0 2024-09-21T12:29:51.936 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: format_version: 5 2024-09-21T12:29:51.936 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: enable_index_compression: 1 2024-09-21T12:29:51.936 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_align: 0 2024-09-21T12:29:51.936 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: max_auto_readahead_size: 262144 2024-09-21T12:29:51.936 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: prepopulate_block_cache: 0 2024-09-21T12:29:51.936 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: initial_auto_readahead_size: 8192 2024-09-21T12:29:51.936 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: num_file_reads_for_auto_readahead: 2 2024-09-21T12:29:51.936 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T12:29:51.937 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T12:29:51.937 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.compression: NoCompression 2024-09-21T12:29:51.937 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T12:29:51.937 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T12:29:51.937 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T12:29:51.937 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.num_levels: 7 2024-09-21T12:29:51.937 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T12:29:51.937 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T12:29:51.937 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T12:29:51.937 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T12:29:51.937 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T12:29:51.937 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T12:29:51.937 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T12:29:51.937 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T12:29:51.937 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T12:29:51.938 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T12:29:51.938 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-21T12:29:51.938 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-21T12:29:51.938 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T12:29:51.938 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T12:29:51.938 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T12:29:51.938 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T12:29:51.938 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T12:29:51.938 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-21T12:29:51.938 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T12:29:51.938 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T12:29:51.938 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-21T12:29:51.938 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T12:29:51.939 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T12:29:51.939 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T12:29:51.939 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T12:29:51.939 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T12:29:51.939 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T12:29:51.939 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T12:29:51.939 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T12:29:51.939 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T12:29:51.939 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T12:29:51.940 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T12:29:51.940 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T12:29:51.940 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T12:29:51.940 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T12:29:51.940 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T12:29:51.940 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T12:29:51.940 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T12:29:51.940 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-21T12:29:51.940 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.arena_block_size: 1048576 2024-09-21T12:29:51.940 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T12:29:51.940 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T12:29:51.940 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T12:29:51.940 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T12:29:51.940 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T12:29:51.941 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T12:29:51.941 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T12:29:51.941 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T12:29:51.941 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T12:29:51.941 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T12:29:51.941 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T12:29:51.941 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T12:29:51.941 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T12:29:51.941 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T12:29:51.941 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.inplace_update_support: 0 2024-09-21T12:29:51.941 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T12:29:51.941 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T12:29:51.941 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T12:29:51.941 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T12:29:51.942 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.bloom_locality: 0 2024-09-21T12:29:51.942 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.max_successive_merges: 0 2024-09-21T12:29:51.942 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T12:29:51.942 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T12:29:51.942 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T12:29:51.942 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T12:29:51.942 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.ttl: 2592000 2024-09-21T12:29:51.942 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T12:29:51.942 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-21T12:29:51.942 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-21T12:29:51.942 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.enable_blob_files: false 2024-09-21T12:29:51.942 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.min_blob_size: 0 2024-09-21T12:29:51.942 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T12:29:51.943 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T12:29:51.943 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T12:29:51.943 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T12:29:51.943 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-21T12:29:51.943 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-21T12:29:51.943 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.blob_file_starting_level: 0 2024-09-21T12:29:51.943 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-21T12:29:51.943 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-09-21T12:29:51.943 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T12:29:51.943 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi078/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 11851, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T12:29:51.943 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-09-21T12:29:51.943 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 5f1fcf2f-3390-4361-b4df-a0067ec09e39 2024-09-21T12:29:51.943 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726921791877758, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-09-21T12:29:51.943 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-09-21T12:29:51.944 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726921791887530, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 2143552, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11852, "largest_seqno": 12775, "table_properties": {"data_size": 2138675, "index_size": 2808, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1029, "raw_key_size": 9335, "raw_average_key_size": 23, "raw_value_size": 2129881, "raw_average_value_size": 5311, "num_data_blocks": 130, "num_entries": 401, "num_filter_entries": 401, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726921791, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "5f1fcf2f-3390-4361-b4df-a0067ec09e39", "db_session_id": "KQZ5A6VQPJEZE1O8I7XS", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-09-21T12:29:51.944 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726921791887673, "job": 1, "event": "recovery_finished"} 2024-09-21T12:29:51.944 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-09-21T12:29:51.944 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T12:29:51.944 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi078/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T12:29:51.944 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55d5c0a3ae00 2024-09-21T12:29:51.944 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: DB pointer 0x55d5c0b3c000 2024-09-21T12:29:51.944 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-21T12:29:51.944 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-21T12:29:51.944 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** DB Stats ** 2024-09-21T12:29:51.944 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T12:29:51.944 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T12:29:51.944 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T12:29:51.944 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T12:29:51.945 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T12:29:51.945 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T12:29:51.945 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T12:29:51.945 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-21T12:29:51.945 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** Compaction Stats [default] ** 2024-09-21T12:29:51.945 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T12:29:51.945 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-21T12:29:51.945 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: L0 1/0 2.04 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 405.0 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-21T12:29:51.945 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: L6 1/0 7.20 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-21T12:29:51.945 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Sum 2/0 9.25 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 405.0 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-21T12:29:51.945 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 405.0 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-21T12:29:51.945 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-21T12:29:51.945 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** Compaction Stats [default] ** 2024-09-21T12:29:51.945 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T12:29:51.946 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T12:29:51.946 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 405.0 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-21T12:29:51.946 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-21T12:29:51.946 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-21T12:29:51.946 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-21T12:29:51.946 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T12:29:51.946 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-09-21T12:29:51.947 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T12:29:51.947 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T12:29:51.947 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T12:29:51.947 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T12:29:51.947 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative compaction: 0.00 GB write, 149.30 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T12:29:51.947 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval compaction: 0.00 GB write, 149.30 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T12:29:51.947 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T12:29:51.947 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Block cache BinnedLRUCache@0x55d5c0a0b610#2 capacity: 512.00 MB usage: 3.98 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.3e-05 secs_since: 0 2024-09-21T12:29:51.947 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.16 KB,0.000220537%) IndexBlock(1,2.83 KB,0.000539422%) Misc(1,0.00 KB,0%) 2024-09-21T12:29:51.947 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-21T12:29:51.947 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T12:29:51.947 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: starting mon.smithi078 rank 0 at public addrs [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] at bind addrs [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi078 fsid 6263deca-7813-11ef-baf6-efdc52797490 2024-09-21T12:29:51.947 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: mon.smithi078@-1(???) e2 preinit fsid 6263deca-7813-11ef-baf6-efdc52797490 2024-09-21T12:29:51.947 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: mon.smithi078@-1(???).mds e7 new map 2024-09-21T12:29:51.947 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: mon.smithi078@-1(???).mds e7 print_map 2024-09-21T12:29:51.948 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: e7 2024-09-21T12:29:51.948 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-21T12:29:51.948 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T12:29:51.948 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T12:29:51.948 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: legacy client fscid: 1 2024-09-21T12:29:51.948 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-21T12:29:51.948 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Filesystem 'foofs' (1) 2024-09-21T12:29:51.948 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: fs_name foofs 2024-09-21T12:29:51.948 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: epoch 5 2024-09-21T12:29:51.948 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-21T12:29:51.948 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: created 2024-09-21T12:24:45.236210+0000 2024-09-21T12:29:51.948 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: modified 2024-09-21T12:24:51.265131+0000 2024-09-21T12:29:51.948 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: tableserver 0 2024-09-21T12:29:51.948 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: root 0 2024-09-21T12:29:51.949 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: session_timeout 60 2024-09-21T12:29:51.949 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: session_autoclose 300 2024-09-21T12:29:51.949 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: max_file_size 1099511627776 2024-09-21T12:29:51.949 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: max_xattr_size 65536 2024-09-21T12:29:51.949 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: required_client_features {} 2024-09-21T12:29:51.949 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: last_failure 0 2024-09-21T12:29:51.949 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: last_failure_osd_epoch 0 2024-09-21T12:29:51.949 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T12:29:51.949 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: max_mds 1 2024-09-21T12:29:51.949 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: in 0 2024-09-21T12:29:51.949 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: up {0=14494} 2024-09-21T12:29:51.949 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: failed 2024-09-21T12:29:51.949 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: damaged 2024-09-21T12:29:51.950 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: stopped 2024-09-21T12:29:51.950 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: data_pools [3] 2024-09-21T12:29:51.950 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: metadata_pool 2 2024-09-21T12:29:51.950 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: inline_data disabled 2024-09-21T12:29:51.950 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: balancer 2024-09-21T12:29:51.950 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: bal_rank_mask -1 2024-09-21T12:29:51.950 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: standby_count_wanted 1 2024-09-21T12:29:51.950 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: qdb_cluster leader: 0 members: 2024-09-21T12:29:51.950 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: [mds.foofs.smithi078.ytpzwh{0:14494} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.78:6834/169294426,v1:172.21.15.78:6835/169294426] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T12:29:51.950 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-21T12:29:51.950 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-21T12:29:51.950 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Standby daemons: 2024-09-21T12:29:51.950 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-09-21T12:29:51.951 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: [mds.foofs.smithi148.uukezr{-1:24267} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.148:6832/3237004392,v1:172.21.15.148:6833/3237004392] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T12:29:51.951 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: mon.smithi078@-1(???).osd e71 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-21T12:29:51.951 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: mon.smithi078@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T12:29:51.951 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: mon.smithi078@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T12:29:51.951 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: mon.smithi078@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T12:29:51.951 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 ceph-mon[63866]: mon.smithi078@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-09-21T12:29:52.183 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 bash[63848]: 22b3097efbd8c02c4212806c49ca77f908da21e3a296e763947f8da487f3b6db 2024-09-21T12:29:52.184 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:51 smithi078 systemd[1]: Started Ceph mon.smithi078 for 6263deca-7813-11ef-baf6-efdc52797490. 2024-09-21T12:29:52.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 119125 10.65 MB/sec execute 150 sec latency 57.183 ms 2024-09-21T12:29:52.946 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:52 smithi078 ceph-mon[63866]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-21T12:29:52.946 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:52 smithi078 ceph-mon[63866]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi148"}]: dispatch 2024-09-21T12:29:52.946 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:52 smithi078 ceph-mon[63866]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T12:29:52.946 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:52 smithi078 ceph-mon[63866]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T12:29:52.946 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:52 smithi078 ceph-mon[63866]: pgmap v27: 97 pgs: 97 active+clean; 342 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.0 MiB/s wr, 564 op/s 2024-09-21T12:29:52.946 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:52 smithi078 ceph-mon[63866]: mon.smithi078 calling monitor election 2024-09-21T12:29:52.947 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:52 smithi078 ceph-mon[63866]: mon.smithi078 is new leader, mons smithi078,smithi148 in quorum (ranks 0,1) 2024-09-21T12:29:52.947 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:52 smithi078 ceph-mon[63866]: monmap epoch 2 2024-09-21T12:29:52.947 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:52 smithi078 ceph-mon[63866]: fsid 6263deca-7813-11ef-baf6-efdc52797490 2024-09-21T12:29:52.947 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:52 smithi078 ceph-mon[63866]: last_changed 2024-09-21T12:19:32.763829+0000 2024-09-21T12:29:52.947 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:52 smithi078 ceph-mon[63866]: created 2024-09-21T12:17:23.120662+0000 2024-09-21T12:29:52.947 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:52 smithi078 ceph-mon[63866]: min_mon_release 17 (quincy) 2024-09-21T12:29:52.947 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:52 smithi078 ceph-mon[63866]: election_strategy: 1 2024-09-21T12:29:52.947 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:52 smithi078 ceph-mon[63866]: 0: [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon.smithi078 2024-09-21T12:29:52.947 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:52 smithi078 ceph-mon[63866]: 1: [v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0] mon.smithi148 2024-09-21T12:29:52.947 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:52 smithi078 ceph-mon[63866]: fsmap foofs:1 {0=foofs.smithi078.ytpzwh=up:active} 1 up:standby 2024-09-21T12:29:52.947 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:52 smithi078 ceph-mon[63866]: osdmap e71: 8 total, 8 up, 8 in 2024-09-21T12:29:52.948 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:52 smithi078 ceph-mon[63866]: mgrmap e35: smithi078.ntwyan(active, since 51s), standbys: smithi148.oyiqmx 2024-09-21T12:29:52.948 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:52 smithi078 ceph-mon[63866]: overall HEALTH_OK 2024-09-21T12:29:52.948 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:52 smithi078 ceph-mon[63866]: from='mgr.14788 ' entity='' 2024-09-21T12:29:52.948 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:52 smithi078 ceph-mon[63866]: mgrmap e36: smithi078.ntwyan(active, since 51s), standbys: smithi148.oyiqmx 2024-09-21T12:29:53.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:52 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-09-21T12:29:53.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:52 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "mon metadata", "id": "smithi148"}]: dispatch 2024-09-21T12:29:53.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:52 smithi148 ceph-mon[25818]: from='mgr.14788 172.21.15.78:0/1385412818' entity='mgr.smithi078.ntwyan' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T12:29:53.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:52 smithi148 ceph-mon[25818]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T12:29:53.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:52 smithi148 ceph-mon[25818]: pgmap v27: 97 pgs: 97 active+clean; 342 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.0 MiB/s wr, 564 op/s 2024-09-21T12:29:53.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:52 smithi148 ceph-mon[25818]: mon.smithi078 calling monitor election 2024-09-21T12:29:53.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:52 smithi148 ceph-mon[25818]: mon.smithi078 is new leader, mons smithi078,smithi148 in quorum (ranks 0,1) 2024-09-21T12:29:53.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:52 smithi148 ceph-mon[25818]: monmap epoch 2 2024-09-21T12:29:53.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:52 smithi148 ceph-mon[25818]: fsid 6263deca-7813-11ef-baf6-efdc52797490 2024-09-21T12:29:53.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:52 smithi148 ceph-mon[25818]: last_changed 2024-09-21T12:19:32.763829+0000 2024-09-21T12:29:53.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:52 smithi148 ceph-mon[25818]: created 2024-09-21T12:17:23.120662+0000 2024-09-21T12:29:53.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:52 smithi148 ceph-mon[25818]: min_mon_release 17 (quincy) 2024-09-21T12:29:53.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:52 smithi148 ceph-mon[25818]: election_strategy: 1 2024-09-21T12:29:53.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:52 smithi148 ceph-mon[25818]: 0: [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon.smithi078 2024-09-21T12:29:53.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:52 smithi148 ceph-mon[25818]: 1: [v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0] mon.smithi148 2024-09-21T12:29:53.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:52 smithi148 ceph-mon[25818]: fsmap foofs:1 {0=foofs.smithi078.ytpzwh=up:active} 1 up:standby 2024-09-21T12:29:53.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:52 smithi148 ceph-mon[25818]: osdmap e71: 8 total, 8 up, 8 in 2024-09-21T12:29:53.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:52 smithi148 ceph-mon[25818]: mgrmap e35: smithi078.ntwyan(active, since 51s), standbys: smithi148.oyiqmx 2024-09-21T12:29:53.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:52 smithi148 ceph-mon[25818]: overall HEALTH_OK 2024-09-21T12:29:53.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:52 smithi148 ceph-mon[25818]: from='mgr.14788 ' entity='' 2024-09-21T12:29:53.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:52 smithi148 ceph-mon[25818]: mgrmap e36: smithi078.ntwyan(active, since 51s), standbys: smithi148.oyiqmx 2024-09-21T12:29:53.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 119487 10.66 MB/sec execute 151 sec latency 57.414 ms 2024-09-21T12:29:54.737 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:54 smithi078 ceph-mon[63866]: mgrmap e37: smithi078.ntwyan(active, since 52s), standbys: smithi148.oyiqmx 2024-09-21T12:29:54.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 119995 10.73 MB/sec execute 152 sec latency 56.744 ms 2024-09-21T12:29:55.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:54 smithi148 ceph-mon[25818]: mgrmap e37: smithi078.ntwyan(active, since 52s), standbys: smithi148.oyiqmx 2024-09-21T12:29:55.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 120261 10.70 MB/sec execute 153 sec latency 43.679 ms 2024-09-21T12:29:56.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 120698 10.76 MB/sec execute 154 sec latency 60.694 ms 2024-09-21T12:29:57.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 121137 10.75 MB/sec execute 155 sec latency 41.000 ms 2024-09-21T12:29:58.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 121415 10.70 MB/sec execute 156 sec latency 46.668 ms 2024-09-21T12:29:59.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:59 smithi148 ceph-mon[25818]: from='mgr.? 172.21.15.148:0/2112191211' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi148.oyiqmx/crt"}]: dispatch 2024-09-21T12:29:59.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:59 smithi148 ceph-mon[25818]: from='mgr.? 172.21.15.148:0/2112191211' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T12:29:59.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:59 smithi148 ceph-mon[25818]: from='mgr.? 172.21.15.148:0/2112191211' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi148.oyiqmx/key"}]: dispatch 2024-09-21T12:29:59.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:59 smithi148 ceph-mon[25818]: from='mgr.? 172.21.15.148:0/2112191211' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T12:29:59.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:59 smithi148 ceph-mon[25818]: Standby manager daemon smithi148.oyiqmx restarted 2024-09-21T12:29:59.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:29:59 smithi148 ceph-mon[25818]: Standby manager daemon smithi148.oyiqmx started 2024-09-21T12:29:59.340 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:59 smithi078 ceph-mon[63866]: from='mgr.? 172.21.15.148:0/2112191211' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi148.oyiqmx/crt"}]: dispatch 2024-09-21T12:29:59.341 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:59 smithi078 ceph-mon[63866]: from='mgr.? 172.21.15.148:0/2112191211' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T12:29:59.341 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:59 smithi078 ceph-mon[63866]: from='mgr.? 172.21.15.148:0/2112191211' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi148.oyiqmx/key"}]: dispatch 2024-09-21T12:29:59.341 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:59 smithi078 ceph-mon[63866]: from='mgr.? 172.21.15.148:0/2112191211' entity='mgr.smithi148.oyiqmx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T12:29:59.341 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:59 smithi078 ceph-mon[63866]: Standby manager daemon smithi148.oyiqmx restarted 2024-09-21T12:29:59.341 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:29:59 smithi078 ceph-mon[63866]: Standby manager daemon smithi148.oyiqmx started 2024-09-21T12:29:59.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 121719 10.68 MB/sec execute 157 sec latency 54.439 ms 2024-09-21T12:30:00.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:00 smithi078 ceph-mon[63866]: mgrmap e38: smithi078.ntwyan(active, since 57s), standbys: smithi148.oyiqmx 2024-09-21T12:30:00.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:00 smithi078 ceph-mon[63866]: Active manager daemon smithi078.ntwyan restarted 2024-09-21T12:30:00.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:00 smithi078 ceph-mon[63866]: Activating manager daemon smithi078.ntwyan 2024-09-21T12:30:00.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:00 smithi078 ceph-mon[63866]: osdmap e72: 8 total, 8 up, 8 in 2024-09-21T12:30:00.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:00 smithi078 ceph-mon[63866]: mgrmap e39: smithi078.ntwyan(active, starting, since 0.018643s), standbys: smithi148.oyiqmx 2024-09-21T12:30:00.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mon metadata", "id": "smithi078"} : dispatch 2024-09-21T12:30:00.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mon metadata", "id": "smithi148"} : dispatch 2024-09-21T12:30:00.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mds metadata", "who": "foofs.smithi078.ytpzwh"} : dispatch 2024-09-21T12:30:00.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mds metadata", "who": "foofs.smithi148.uukezr"} : dispatch 2024-09-21T12:30:00.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mgr metadata", "who": "smithi078.ntwyan", "id": "smithi078.ntwyan"} : dispatch 2024-09-21T12:30:00.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mgr metadata", "who": "smithi148.oyiqmx", "id": "smithi148.oyiqmx"} : dispatch 2024-09-21T12:30:00.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T12:30:00.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T12:30:00.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T12:30:00.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T12:30:00.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T12:30:00.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T12:30:00.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-21T12:30:00.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-21T12:30:00.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mds metadata"} : dispatch 2024-09-21T12:30:00.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata"} : dispatch 2024-09-21T12:30:00.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mon metadata"} : dispatch 2024-09-21T12:30:00.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:00 smithi078 ceph-mon[63866]: Manager daemon smithi078.ntwyan is now available 2024-09-21T12:30:00.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:30:00.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:30:00.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-21T12:30:00.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/mirror_snapshot_schedule"} : dispatch 2024-09-21T12:30:00.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/trash_purge_schedule"} : dispatch 2024-09-21T12:30:00.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:00 smithi078 ceph-mon[63866]: overall HEALTH_OK 2024-09-21T12:30:00.554 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:00 smithi148 ceph-mon[25818]: mgrmap e38: smithi078.ntwyan(active, since 57s), standbys: smithi148.oyiqmx 2024-09-21T12:30:00.554 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:00 smithi148 ceph-mon[25818]: Active manager daemon smithi078.ntwyan restarted 2024-09-21T12:30:00.554 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:00 smithi148 ceph-mon[25818]: Activating manager daemon smithi078.ntwyan 2024-09-21T12:30:00.555 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:00 smithi148 ceph-mon[25818]: osdmap e72: 8 total, 8 up, 8 in 2024-09-21T12:30:00.555 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:00 smithi148 ceph-mon[25818]: mgrmap e39: smithi078.ntwyan(active, starting, since 0.018643s), standbys: smithi148.oyiqmx 2024-09-21T12:30:00.555 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:00 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mon metadata", "id": "smithi078"} : dispatch 2024-09-21T12:30:00.555 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:00 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mon metadata", "id": "smithi148"} : dispatch 2024-09-21T12:30:00.555 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:00 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mds metadata", "who": "foofs.smithi078.ytpzwh"} : dispatch 2024-09-21T12:30:00.555 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:00 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mds metadata", "who": "foofs.smithi148.uukezr"} : dispatch 2024-09-21T12:30:00.555 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:00 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mgr metadata", "who": "smithi078.ntwyan", "id": "smithi078.ntwyan"} : dispatch 2024-09-21T12:30:00.555 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:00 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mgr metadata", "who": "smithi148.oyiqmx", "id": "smithi148.oyiqmx"} : dispatch 2024-09-21T12:30:00.555 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:00 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T12:30:00.555 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:00 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T12:30:00.555 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:00 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T12:30:00.555 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:00 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T12:30:00.555 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:00 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T12:30:00.556 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:00 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T12:30:00.556 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:00 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-21T12:30:00.556 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:00 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-21T12:30:00.556 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:00 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mds metadata"} : dispatch 2024-09-21T12:30:00.556 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:00 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata"} : dispatch 2024-09-21T12:30:00.556 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:00 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mon metadata"} : dispatch 2024-09-21T12:30:00.556 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:00 smithi148 ceph-mon[25818]: Manager daemon smithi078.ntwyan is now available 2024-09-21T12:30:00.556 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:00 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:30:00.556 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:00 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:30:00.556 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:00 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-21T12:30:00.556 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:00 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/mirror_snapshot_schedule"} : dispatch 2024-09-21T12:30:00.556 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:00 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.ntwyan/trash_purge_schedule"} : dispatch 2024-09-21T12:30:00.556 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:00 smithi148 ceph-mon[25818]: overall HEALTH_OK 2024-09-21T12:30:00.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 122018 10.66 MB/sec execute 158 sec latency 88.960 ms 2024-09-21T12:30:01.558 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:01 smithi078 ceph-mon[63866]: mgrmap e40: smithi078.ntwyan(active, since 1.05767s), standbys: smithi148.oyiqmx 2024-09-21T12:30:01.558 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:01 smithi078 ceph-mon[63866]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T12:30:01.558 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:01 smithi078 ceph-mon[63866]: pgmap v3: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T12:30:01.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 122503 10.64 MB/sec execute 159 sec latency 50.653 ms 2024-09-21T12:30:01.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:01 smithi148 ceph-mon[25818]: mgrmap e40: smithi078.ntwyan(active, since 1.05767s), standbys: smithi148.oyiqmx 2024-09-21T12:30:01.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:01 smithi148 ceph-mon[25818]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T12:30:01.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:01 smithi148 ceph-mon[25818]: pgmap v3: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T12:30:02.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:02 smithi078 ceph-mon[63866]: [21/Sep/2024:12:30:01] ENGINE Bus STARTING 2024-09-21T12:30:02.583 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:02 smithi078 ceph-mon[63866]: [21/Sep/2024:12:30:01] ENGINE Serving on http://172.21.15.78:8765 2024-09-21T12:30:02.584 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:02 smithi078 ceph-mon[63866]: [21/Sep/2024:12:30:01] ENGINE Serving on https://172.21.15.78:7150 2024-09-21T12:30:02.584 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:02 smithi078 ceph-mon[63866]: [21/Sep/2024:12:30:01] ENGINE Bus STARTED 2024-09-21T12:30:02.584 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:02 smithi078 ceph-mon[63866]: [21/Sep/2024:12:30:01] ENGINE Error in HTTPServer.serve 2024-09-21T12:30:02.584 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Traceback (most recent call last): 2024-09-21T12:30:02.584 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T12:30:02.584 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self._connections.run(self.expiration_interval) 2024-09-21T12:30:02.584 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T12:30:02.584 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self._run(expiration_interval) 2024-09-21T12:30:02.584 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T12:30:02.584 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T12:30:02.584 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T12:30:02.585 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T12:30:02.585 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T12:30:02.585 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: s = self.context.wrap_socket( 2024-09-21T12:30:02.585 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T12:30:02.585 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: return self.sslsocket_class._create( 2024-09-21T12:30:02.585 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T12:30:02.585 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self.do_handshake() 2024-09-21T12:30:02.585 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T12:30:02.585 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: self._sslobj.do_handshake() 2024-09-21T12:30:02.585 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T12:30:02.585 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:02 smithi078 ceph-mon[63866]: pgmap v4: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T12:30:02.585 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:02 smithi078 ceph-mon[63866]: mgrmap e41: smithi078.ntwyan(active, since 2s), standbys: smithi148.oyiqmx 2024-09-21T12:30:02.779 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:02 smithi148 ceph-mon[25818]: [21/Sep/2024:12:30:01] ENGINE Bus STARTING 2024-09-21T12:30:02.779 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:02 smithi148 ceph-mon[25818]: [21/Sep/2024:12:30:01] ENGINE Serving on http://172.21.15.78:8765 2024-09-21T12:30:02.779 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:02 smithi148 ceph-mon[25818]: [21/Sep/2024:12:30:01] ENGINE Serving on https://172.21.15.78:7150 2024-09-21T12:30:02.779 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:02 smithi148 ceph-mon[25818]: [21/Sep/2024:12:30:01] ENGINE Bus STARTED 2024-09-21T12:30:02.779 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:02 smithi148 ceph-mon[25818]: [21/Sep/2024:12:30:01] ENGINE Error in HTTPServer.serve 2024-09-21T12:30:02.779 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Traceback (most recent call last): 2024-09-21T12:30:02.779 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T12:30:02.780 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: self._connections.run(self.expiration_interval) 2024-09-21T12:30:02.780 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T12:30:02.780 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: self._run(expiration_interval) 2024-09-21T12:30:02.780 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T12:30:02.780 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T12:30:02.780 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T12:30:02.780 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T12:30:02.780 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T12:30:02.780 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: s = self.context.wrap_socket( 2024-09-21T12:30:02.780 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T12:30:02.780 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: return self.sslsocket_class._create( 2024-09-21T12:30:02.780 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T12:30:02.780 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: self.do_handshake() 2024-09-21T12:30:02.780 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T12:30:02.780 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: self._sslobj.do_handshake() 2024-09-21T12:30:02.781 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T12:30:02.781 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:02 smithi148 ceph-mon[25818]: pgmap v4: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T12:30:02.781 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:02 smithi148 ceph-mon[25818]: mgrmap e41: smithi078.ntwyan(active, since 2s), standbys: smithi148.oyiqmx 2024-09-21T12:30:02.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 122956 10.65 MB/sec execute 160 sec latency 56.206 ms 2024-09-21T12:30:03.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 123296 10.64 MB/sec execute 161 sec latency 39.574 ms 2024-09-21T12:30:04.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:04 smithi078 ceph-mon[63866]: pgmap v5: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T12:30:04.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:04 smithi078 ceph-mon[63866]: mgrmap e42: smithi078.ntwyan(active, since 4s), standbys: smithi148.oyiqmx 2024-09-21T12:30:04.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:04 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:04.624 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:04 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:04.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 123660 10.67 MB/sec execute 162 sec latency 43.329 ms 2024-09-21T12:30:04.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:04 smithi148 ceph-mon[25818]: pgmap v5: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T12:30:04.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:04 smithi148 ceph-mon[25818]: mgrmap e42: smithi078.ntwyan(active, since 4s), standbys: smithi148.oyiqmx 2024-09-21T12:30:04.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:04 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:04.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:04 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:05.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 123940 10.66 MB/sec execute 163 sec latency 45.147 ms 2024-09-21T12:30:05.925 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:05 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:05.926 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:05 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:06.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:05 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:06.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:05 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:06.725 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:06 smithi078 ceph-mon[63866]: pgmap v6: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T12:30:06.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 124180 10.66 MB/sec execute 164 sec latency 66.785 ms 2024-09-21T12:30:06.907 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:06 smithi148 ceph-mon[25818]: pgmap v6: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-21T12:30:07.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 124438 10.64 MB/sec execute 165 sec latency 82.141 ms 2024-09-21T12:30:08.765 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:08 smithi078 ceph-mon[63866]: pgmap v7: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 30 KiB/s rd, 4.1 MiB/s wr, 395 op/s 2024-09-21T12:30:08.765 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:08 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:08.765 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:08 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:08.789 INFO:teuthology.orchestra.run.smithi078.stdout: 5 124640 10.61 MB/sec execute 166 sec latency 59.264 ms 2024-09-21T12:30:09.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:08 smithi148 ceph-mon[25818]: pgmap v7: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 30 KiB/s rd, 4.1 MiB/s wr, 395 op/s 2024-09-21T12:30:09.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:08 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:09.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:08 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:09.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 124854 10.56 MB/sec execute 167 sec latency 72.997 ms 2024-09-21T12:30:10.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:10 smithi078 ceph-mon[63866]: pgmap v8: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 3.2 MiB/s wr, 306 op/s 2024-09-21T12:30:10.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:10 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:10.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:10 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:10.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:10 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:10.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:10 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:10.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:10 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:10.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:10 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:30:10.790 INFO:teuthology.orchestra.run.smithi078.stdout: 5 125071 10.53 MB/sec execute 168 sec latency 62.038 ms 2024-09-21T12:30:10.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:10 smithi148 ceph-mon[25818]: pgmap v8: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 3.2 MiB/s wr, 306 op/s 2024-09-21T12:30:10.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:10 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:10.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:10 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:10.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:10 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:10.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:10 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:10.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:10 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:10.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:10 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:30:11.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:11 smithi078 ceph-mon[63866]: Updating smithi078:/etc/ceph/ceph.conf 2024-09-21T12:30:11.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:11 smithi078 ceph-mon[63866]: Updating smithi148:/etc/ceph/ceph.conf 2024-09-21T12:30:11.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:11 smithi078 ceph-mon[63866]: Updating smithi148:/var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T12:30:11.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:11 smithi078 ceph-mon[63866]: Updating smithi078:/var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T12:30:11.571 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:11 smithi078 ceph-mon[63866]: Updating smithi148:/etc/ceph/ceph.client.admin.keyring 2024-09-21T12:30:11.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:11 smithi078 ceph-mon[63866]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-09-21T12:30:11.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:11 smithi078 ceph-mon[63866]: Updating smithi148:/var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T12:30:11.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:11 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:11.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:11 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:11.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:11 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:11.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:11 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:11.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:11 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:11.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:11 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:11.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:11 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:30:11.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:11 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:30:11.572 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:11 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "quorum_status"} : dispatch 2024-09-21T12:30:11.798 INFO:teuthology.orchestra.run.smithi078.stdout: 5 125383 10.49 MB/sec execute 169 sec latency 59.190 ms 2024-09-21T12:30:11.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:11 smithi148 ceph-mon[25818]: Updating smithi078:/etc/ceph/ceph.conf 2024-09-21T12:30:11.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:11 smithi148 ceph-mon[25818]: Updating smithi148:/etc/ceph/ceph.conf 2024-09-21T12:30:11.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:11 smithi148 ceph-mon[25818]: Updating smithi148:/var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T12:30:11.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:11 smithi148 ceph-mon[25818]: Updating smithi078:/var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T12:30:11.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:11 smithi148 ceph-mon[25818]: Updating smithi148:/etc/ceph/ceph.client.admin.keyring 2024-09-21T12:30:11.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:11 smithi148 ceph-mon[25818]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-09-21T12:30:11.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:11 smithi148 ceph-mon[25818]: Updating smithi148:/var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T12:30:11.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:11 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:11.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:11 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:11.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:11 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:11.802 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:11 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:11.802 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:11 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:11.802 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:11 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:11.802 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:11 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:30:11.802 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:11 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:30:11.802 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:11 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "quorum_status"} : dispatch 2024-09-21T12:30:12.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:12 smithi078 ceph-mon[63866]: Updating smithi078:/var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T12:30:12.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:12 smithi078 ceph-mon[63866]: pgmap v9: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 22 KiB/s rd, 4.3 MiB/s wr, 412 op/s 2024-09-21T12:30:12.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 125747 10.48 MB/sec execute 170 sec latency 54.322 ms 2024-09-21T12:30:12.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:12 smithi148 ceph-mon[25818]: Updating smithi078:/var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T12:30:12.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:12 smithi148 ceph-mon[25818]: pgmap v9: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 22 KiB/s rd, 4.3 MiB/s wr, 412 op/s 2024-09-21T12:30:13.493 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-21T12:30:13.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 126129 10.46 MB/sec execute 171 sec latency 71.535 ms 2024-09-21T12:30:14.376 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T12:30:14.376 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (2m) 4s ago 11m 20.7M - 0.27.0 11f11916f8cd e6e67ab07dec 2024-09-21T12:30:14.376 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (11m) 4s ago 11m 6400k - 17.2.0 e1d6a67b021e 4a209e5da5ce 2024-09-21T12:30:14.376 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi148 smithi148 running (10m) 10s ago 10m 6405k - 17.2.0 e1d6a67b021e 0fa5e612307f 2024-09-21T12:30:14.377 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (2m) 4s ago 11m 53.3M - dad864ee21e9 2a140c36091b 2024-09-21T12:30:14.377 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.ytpzwh smithi078 running (5m) 4s ago 5m 37.4M - 17.2.0 e1d6a67b021e 5e9b109502a2 2024-09-21T12:30:14.377 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi148.uukezr smithi148 running (5m) 10s ago 5m 13.3M - 17.2.0 e1d6a67b021e b7e8aa0f3a6c 2024-09-21T12:30:14.377 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.ntwyan smithi078 *:8443,9283,8765 running (91s) 4s ago 12m 488M - 19.3.0-5104-g44178494 b415816db8b4 a61baedebf43 2024-09-21T12:30:14.377 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi148.oyiqmx smithi148 *:8443,9283,8765 running (40s) 10s ago 10m 451M - 19.3.0-5104-g44178494 b415816db8b4 0255535c6fdb 2024-09-21T12:30:14.377 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (23s) 4s ago 12m 34.4M 2048M 19.3.0-5104-g44178494 b415816db8b4 22b3097efbd8 2024-09-21T12:30:14.377 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi148 smithi148 running (10m) 10s ago 10m 50.0M 2048M 17.2.0 e1d6a67b021e fda961db230f 2024-09-21T12:30:14.377 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.ydbhru smithi078 *:2049 running (5m) 4s ago 5m 262M - 4.0 e1d6a67b021e 7e5b0274ac38 2024-09-21T12:30:14.377 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi148.einuzt smithi148 *:2049 running (4m) 10s ago 4m 85.8M - 4.0 e1d6a67b021e 76a8b0eabede 2024-09-21T12:30:14.377 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (2m) 4s ago 11m 11.5M - 1.7.0 72c9c2088986 301adb4e9e22 2024-09-21T12:30:14.377 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi148 smithi148 *:9100 running (102s) 10s ago 10m 9965k - 1.7.0 72c9c2088986 0cceff4bf342 2024-09-21T12:30:14.377 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (9m) 4s ago 9m 453M 1936M 17.2.0 e1d6a67b021e d4d5e9977070 2024-09-21T12:30:14.377 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (9m) 4s ago 9m 402M 1936M 17.2.0 e1d6a67b021e b9a4b952233b 2024-09-21T12:30:14.377 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (8m) 4s ago 8m 308M 1936M 17.2.0 e1d6a67b021e 829fb8c2d827 2024-09-21T12:30:14.378 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (8m) 4s ago 8m 416M 1936M 17.2.0 e1d6a67b021e ef942c8556d0 2024-09-21T12:30:14.378 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi148 running (8m) 10s ago 8m 425M 2704M 17.2.0 e1d6a67b021e 62eefdd572c0 2024-09-21T12:30:14.378 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi148 running (7m) 10s ago 7m 470M 2704M 17.2.0 e1d6a67b021e 7b44be78baca 2024-09-21T12:30:14.378 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi148 running (7m) 10s ago 7m 412M 2704M 17.2.0 e1d6a67b021e 3d23057910ea 2024-09-21T12:30:14.378 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi148 running (6m) 10s ago 6m 517M 2704M 17.2.0 e1d6a67b021e f9e4db9d35fa 2024-09-21T12:30:14.378 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (46s) 4s ago 10m 47.4M - 2.51.0 1d3b7f56885b face2d4cdd1b 2024-09-21T12:30:14.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:14 smithi078 ceph-mon[63866]: Upgrade: Updating mon.smithi148 2024-09-21T12:30:14.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:14 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:14.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:14 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T12:30:14.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:14 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T12:30:14.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:14 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:14.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:14 smithi078 ceph-mon[63866]: Deploying daemon mon.smithi148 on smithi148 2024-09-21T12:30:14.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:14 smithi078 ceph-mon[63866]: pgmap v10: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 20 KiB/s rd, 3.9 MiB/s wr, 373 op/s 2024-09-21T12:30:14.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:14 smithi078 ceph-mon[63866]: from='client.34126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:30:14.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:14 smithi148 ceph-mon[25818]: Upgrade: Updating mon.smithi148 2024-09-21T12:30:14.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:14 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:14.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:14 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T12:30:14.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:14 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T12:30:14.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:14 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:14.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:14 smithi148 ceph-mon[25818]: Deploying daemon mon.smithi148 on smithi148 2024-09-21T12:30:14.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:14 smithi148 ceph-mon[25818]: pgmap v10: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 20 KiB/s rd, 3.9 MiB/s wr, 373 op/s 2024-09-21T12:30:14.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:14 smithi148 ceph-mon[25818]: from='client.34126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:30:14.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 126526 10.46 MB/sec execute 172 sec latency 72.790 ms 2024-09-21T12:30:14.957 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:30:14.957 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-21T12:30:14.958 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-21T12:30:14.958 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 1 2024-09-21T12:30:14.958 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:30:14.958 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-21T12:30:14.958 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:30:14.958 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:30:14.958 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-21T12:30:14.958 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T12:30:14.958 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:30:14.958 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-21T12:30:14.958 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:30:14.958 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:30:14.958 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-21T12:30:14.958 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-09-21T12:30:14.959 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 3 2024-09-21T12:30:14.959 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T12:30:14.959 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:30:15.159 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:15 smithi078 ceph-mon[63866]: from='client.34130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:30:15.160 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:15 smithi078 ceph-mon[63866]: from='client.34134 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:30:15.160 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:15 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:30:15.160 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:15 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/2818894451' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:30:15.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:15 smithi148 systemd[1]: Stopping Ceph mon.smithi148 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:30:15.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:15 smithi148 ceph-mon[25818]: from='client.34130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:30:15.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:15 smithi148 ceph-mon[25818]: from='client.34134 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:30:15.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:15 smithi148 ceph-mon[25818]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:30:15.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:15 smithi148 ceph-mon[25818]: from='client.? 172.21.15.78:0/2818894451' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:30:15.510 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:30:15.510 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5", 2024-09-21T12:30:15.510 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-21T12:30:15.510 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T12:30:15.510 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-21T12:30:15.510 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr" 2024-09-21T12:30:15.510 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-21T12:30:15.510 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "3/23 daemons upgraded", 2024-09-21T12:30:15.510 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading mon daemons", 2024-09-21T12:30:15.511 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-21T12:30:15.511 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:30:15.684 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:15 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi148[25812]: 2024-09-21T12:30:15.332+0000 7f265b8ee700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi148 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T12:30:15.685 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:15 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi148[25812]: 2024-09-21T12:30:15.332+0000 7f265b8ee700 -1 mon.smithi148@1(peon) e2 *** Got Signal Terminated *** 2024-09-21T12:30:15.685 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:15 smithi148 podman[49230]: 2024-09-21 12:30:15.373569486 +0000 UTC m=+0.177358772 container died fda961db230f48e5e21a74a156bb3cd9e8976d6341334f63421eb9aeb8f698fb (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi148, io.openshift.expose-services=, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, version=8, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, vcs-type=git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, release=754) 2024-09-21T12:30:15.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 126977 10.49 MB/sec execute 173 sec latency 37.852 ms 2024-09-21T12:30:16.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:15 smithi148 podman[49230]: 2024-09-21 12:30:15.783161107 +0000 UTC m=+0.586950381 container cleanup fda961db230f48e5e21a74a156bb3cd9e8976d6341334f63421eb9aeb8f698fb (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi148, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, release=754, vcs-type=git, version=8, distribution-scope=public, vendor=Red Hat, Inc., architecture=x86_64, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream) 2024-09-21T12:30:16.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:15 smithi148 bash[49230]: ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi148 2024-09-21T12:30:16.219 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-21T12:30:16.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:16 smithi148 podman[49241]: 2024-09-21 12:30:16.193783466 +0000 UTC m=+0.815120097 container remove fda961db230f48e5e21a74a156bb3cd9e8976d6341334f63421eb9aeb8f698fb (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi148, version=8, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, GIT_CLEAN=True, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, release=754, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, vcs-type=git, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, RELEASE=HEAD, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-21T12:30:16.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:16 smithi148 systemd[1]: ceph-6263deca-7813-11ef-baf6-efdc52797490@mon.smithi148.service: Deactivated successfully. 2024-09-21T12:30:16.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:16 smithi148 systemd[1]: Stopped Ceph mon.smithi148 for 6263deca-7813-11ef-baf6-efdc52797490. 2024-09-21T12:30:16.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:16 smithi148 systemd[1]: ceph-6263deca-7813-11ef-baf6-efdc52797490@mon.smithi148.service: Consumed 6.854s CPU time. 2024-09-21T12:30:16.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 127442 10.53 MB/sec execute 174 sec latency 34.800 ms 2024-09-21T12:30:16.974 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:16 smithi148 systemd[1]: Starting Ceph mon.smithi148 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:30:16.974 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:16 smithi148 podman[49369]: 2024-09-21 12:30:16.725432257 +0000 UTC m=+0.016475158 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:30:16.974 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:16 smithi148 podman[49369]: 2024-09-21 12:30:16.83683657 +0000 UTC m=+0.127879456 container create df8b0850913677c8d686548f8c3522af5680c9db06e6cd852dd2ed1ec5d0a17a (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi148, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.license=GPLv2) 2024-09-21T12:30:17.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 podman[49369]: 2024-09-21 12:30:17.106592992 +0000 UTC m=+0.397635891 container init df8b0850913677c8d686548f8c3522af5680c9db06e6cd852dd2ed1ec5d0a17a (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi148, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T12:30:17.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 podman[49369]: 2024-09-21 12:30:17.11026589 +0000 UTC m=+0.401308785 container start df8b0850913677c8d686548f8c3522af5680c9db06e6cd852dd2ed1ec5d0a17a (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi148, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-21T12:30:17.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T12:30:17.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev), process ceph-mon, pid 2 2024-09-21T12:30:17.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: pidfile_write: ignore empty --pid-file 2024-09-21T12:30:17.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: load: jerasure load: lrc 2024-09-21T12:30:17.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: RocksDB version: 7.9.2 2024-09-21T12:30:17.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Git sha 0 2024-09-21T12:30:17.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Compile date 2024-09-21 09:24:23 2024-09-21T12:30:17.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: DB SUMMARY 2024-09-21T12:30:17.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: DB Session ID: MR1RO5LVL13HC9XIJP9U 2024-09-21T12:30:17.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: CURRENT file: CURRENT 2024-09-21T12:30:17.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T12:30:17.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: MANIFEST file: MANIFEST-000009 size: 904 Bytes 2024-09-21T12:30:17.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi148/store.db dir, Total Num: 1, files: 000024.sst 2024-09-21T12:30:17.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi148/store.db: 000022.log size: 7006822 ; 2024-09-21T12:30:17.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.error_if_exists: 0 2024-09-21T12:30:17.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.create_if_missing: 0 2024-09-21T12:30:17.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.paranoid_checks: 1 2024-09-21T12:30:17.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-21T12:30:17.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T12:30:17.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-21T12:30:17.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.env: 0x55f633974120 2024-09-21T12:30:17.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.fs: PosixFileSystem 2024-09-21T12:30:17.553 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.info_log: 0x55f6353b38e0 2024-09-21T12:30:17.553 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T12:30:17.554 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.statistics: (nil) 2024-09-21T12:30:17.554 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.use_fsync: 0 2024-09-21T12:30:17.554 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.max_log_file_size: 0 2024-09-21T12:30:17.554 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T12:30:17.554 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T12:30:17.554 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T12:30:17.554 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T12:30:17.554 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.allow_fallocate: 1 2024-09-21T12:30:17.554 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T12:30:17.555 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T12:30:17.555 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.use_direct_reads: 0 2024-09-21T12:30:17.555 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T12:30:17.555 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T12:30:17.555 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.db_log_dir: 2024-09-21T12:30:17.555 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.wal_dir: 2024-09-21T12:30:17.555 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T12:30:17.555 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T12:30:17.555 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T12:30:17.555 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T12:30:17.555 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T12:30:17.555 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T12:30:17.555 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T12:30:17.555 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T12:30:17.556 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.write_buffer_manager: 0x55f6353b7b80 2024-09-21T12:30:17.556 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T12:30:17.556 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T12:30:17.556 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T12:30:17.556 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T12:30:17.556 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T12:30:17.556 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T12:30:17.556 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T12:30:17.556 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T12:30:17.556 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.unordered_write: 0 2024-09-21T12:30:17.556 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T12:30:17.556 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T12:30:17.556 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T12:30:17.557 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T12:30:17.557 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.row_cache: None 2024-09-21T12:30:17.557 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.wal_filter: None 2024-09-21T12:30:17.557 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T12:30:17.557 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T12:30:17.557 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.two_write_queues: 0 2024-09-21T12:30:17.557 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T12:30:17.557 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.wal_compression: 0 2024-09-21T12:30:17.557 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.atomic_flush: 0 2024-09-21T12:30:17.557 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T12:30:17.557 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T12:30:17.557 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T12:30:17.557 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.log_readahead_size: 0 2024-09-21T12:30:17.557 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T12:30:17.558 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T12:30:17.558 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T12:30:17.558 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T12:30:17.558 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T12:30:17.558 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T12:30:17.558 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-21T12:30:17.558 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.max_background_jobs: 2 2024-09-21T12:30:17.558 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.max_background_compactions: -1 2024-09-21T12:30:17.558 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.max_subcompactions: 1 2024-09-21T12:30:17.558 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T12:30:17.558 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T12:30:17.558 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T12:30:17.558 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T12:30:17.558 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T12:30:17.559 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T12:30:17.559 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T12:30:17.559 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T12:30:17.559 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.max_open_files: -1 2024-09-21T12:30:17.559 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T12:30:17.559 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T12:30:17.559 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T12:30:17.559 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T12:30:17.559 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.max_background_flushes: -1 2024-09-21T12:30:17.559 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Compression algorithms supported: 2024-09-21T12:30:17.559 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: kZSTD supported: 0 2024-09-21T12:30:17.559 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: kXpressCompression supported: 0 2024-09-21T12:30:17.559 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: kBZip2Compression supported: 0 2024-09-21T12:30:17.559 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T12:30:17.560 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: kLZ4Compression supported: 1 2024-09-21T12:30:17.560 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: kZlibCompression supported: 1 2024-09-21T12:30:17.560 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T12:30:17.560 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: kSnappyCompression supported: 1 2024-09-21T12:30:17.560 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T12:30:17.560 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-21T12:30:17.560 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi148/store.db/MANIFEST-000009 2024-09-21T12:30:17.560 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-21T12:30:17.560 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T12:30:17.560 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.merge_operator: 2024-09-21T12:30:17.560 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.compaction_filter: None 2024-09-21T12:30:17.560 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T12:30:17.560 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T12:30:17.560 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T12:30:17.561 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T12:30:17.561 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f6353b3540) 2024-09-21T12:30:17.561 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: cache_index_and_filter_blocks: 1 2024-09-21T12:30:17.561 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T12:30:17.561 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T12:30:17.561 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: pin_top_level_index_and_filter: 1 2024-09-21T12:30:17.561 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: index_type: 0 2024-09-21T12:30:17.561 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: data_block_index_type: 0 2024-09-21T12:30:17.561 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: index_shortening: 1 2024-09-21T12:30:17.561 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T12:30:17.561 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: checksum: 4 2024-09-21T12:30:17.561 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: no_block_cache: 0 2024-09-21T12:30:17.561 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: block_cache: 0x55f6353a7610 2024-09-21T12:30:17.561 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: block_cache_name: BinnedLRUCache 2024-09-21T12:30:17.562 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: block_cache_options: 2024-09-21T12:30:17.562 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: capacity : 536870912 2024-09-21T12:30:17.562 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: num_shard_bits : 4 2024-09-21T12:30:17.562 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: strict_capacity_limit : 0 2024-09-21T12:30:17.562 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: high_pri_pool_ratio: 0.000 2024-09-21T12:30:17.562 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: block_cache_compressed: (nil) 2024-09-21T12:30:17.562 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: persistent_cache: (nil) 2024-09-21T12:30:17.562 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: block_size: 4096 2024-09-21T12:30:17.562 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: block_size_deviation: 10 2024-09-21T12:30:17.562 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: block_restart_interval: 16 2024-09-21T12:30:17.562 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: index_block_restart_interval: 1 2024-09-21T12:30:17.562 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: metadata_block_size: 4096 2024-09-21T12:30:17.562 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: partition_filters: 0 2024-09-21T12:30:17.563 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: use_delta_encoding: 1 2024-09-21T12:30:17.563 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: filter_policy: bloomfilter 2024-09-21T12:30:17.563 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: whole_key_filtering: 1 2024-09-21T12:30:17.563 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: verify_compression: 0 2024-09-21T12:30:17.563 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: read_amp_bytes_per_bit: 0 2024-09-21T12:30:17.563 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: format_version: 5 2024-09-21T12:30:17.563 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: enable_index_compression: 1 2024-09-21T12:30:17.563 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: block_align: 0 2024-09-21T12:30:17.563 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: max_auto_readahead_size: 262144 2024-09-21T12:30:17.563 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: prepopulate_block_cache: 0 2024-09-21T12:30:17.563 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: initial_auto_readahead_size: 8192 2024-09-21T12:30:17.563 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: num_file_reads_for_auto_readahead: 2 2024-09-21T12:30:17.563 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T12:30:17.563 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T12:30:17.563 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.compression: NoCompression 2024-09-21T12:30:17.563 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T12:30:17.564 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T12:30:17.564 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T12:30:17.564 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.num_levels: 7 2024-09-21T12:30:17.564 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T12:30:17.564 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T12:30:17.564 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T12:30:17.564 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T12:30:17.564 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T12:30:17.564 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T12:30:17.564 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T12:30:17.564 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T12:30:17.564 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T12:30:17.564 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T12:30:17.564 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-21T12:30:17.564 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-21T12:30:17.565 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T12:30:17.565 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T12:30:17.565 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T12:30:17.565 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T12:30:17.565 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T12:30:17.565 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-21T12:30:17.565 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T12:30:17.565 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T12:30:17.565 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-21T12:30:17.565 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T12:30:17.565 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T12:30:17.565 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T12:30:17.565 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T12:30:17.565 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T12:30:17.565 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T12:30:17.566 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T12:30:17.566 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T12:30:17.566 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T12:30:17.566 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T12:30:17.566 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T12:30:17.566 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T12:30:17.566 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T12:30:17.566 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T12:30:17.566 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T12:30:17.566 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T12:30:17.566 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T12:30:17.566 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-21T12:30:17.566 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.arena_block_size: 1048576 2024-09-21T12:30:17.566 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T12:30:17.567 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T12:30:17.567 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T12:30:17.567 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T12:30:17.567 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T12:30:17.567 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T12:30:17.567 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T12:30:17.567 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T12:30:17.567 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T12:30:17.567 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T12:30:17.567 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T12:30:17.567 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T12:30:17.567 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T12:30:17.567 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T12:30:17.568 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.inplace_update_support: 0 2024-09-21T12:30:17.568 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T12:30:17.568 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T12:30:17.568 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T12:30:17.568 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T12:30:17.568 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.bloom_locality: 0 2024-09-21T12:30:17.568 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.max_successive_merges: 0 2024-09-21T12:30:17.568 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T12:30:17.568 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T12:30:17.568 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T12:30:17.568 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T12:30:17.568 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.ttl: 2592000 2024-09-21T12:30:17.568 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T12:30:17.568 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-21T12:30:17.568 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-21T12:30:17.568 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.enable_blob_files: false 2024-09-21T12:30:17.569 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.min_blob_size: 0 2024-09-21T12:30:17.569 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T12:30:17.569 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T12:30:17.569 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T12:30:17.569 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T12:30:17.569 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-21T12:30:17.569 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-21T12:30:17.569 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.blob_file_starting_level: 0 2024-09-21T12:30:17.569 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-21T12:30:17.569 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 24.sst 2024-09-21T12:30:17.569 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T12:30:17.569 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi148/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 26, last_sequence is 12253, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T12:30:17.569 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2024-09-21T12:30:17.569 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 99a42100-ffb9-4325-b5a5-b454ddf0bb34 2024-09-21T12:30:17.570 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726921817139971, "job": 1, "event": "recovery_started", "wal_files": [22]} 2024-09-21T12:30:17.570 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2024-09-21T12:30:17.570 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726921817155866, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 4259194, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12258, "largest_seqno": 13799, "table_properties": {"data_size": 4252103, "index_size": 4446, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1605, "raw_key_size": 14691, "raw_average_key_size": 23, "raw_value_size": 4238444, "raw_average_value_size": 6738, "num_data_blocks": 201, "num_entries": 629, "num_filter_entries": 629, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726921817, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "99a42100-ffb9-4325-b5a5-b454ddf0bb34", "db_session_id": "MR1RO5LVL13HC9XIJP9U", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2024-09-21T12:30:17.570 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726921817155998, "job": 1, "event": "recovery_finished"} 2024-09-21T12:30:17.570 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2024-09-21T12:30:17.570 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T12:30:17.570 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi148/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T12:30:17.570 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55f6353d6e00 2024-09-21T12:30:17.570 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: DB pointer 0x55f6354d8000 2024-09-21T12:30:17.570 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-21T12:30:17.570 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-21T12:30:17.570 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: ** DB Stats ** 2024-09-21T12:30:17.570 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T12:30:17.570 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T12:30:17.571 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T12:30:17.571 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T12:30:17.571 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T12:30:17.571 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T12:30:17.571 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T12:30:17.571 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: 2024-09-21T12:30:17.571 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: ** Compaction Stats [default] ** 2024-09-21T12:30:17.571 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T12:30:17.571 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-21T12:30:17.571 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: L0 1/0 4.06 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 670.9 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-09-21T12:30:17.571 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: L6 1/0 7.20 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-21T12:30:17.571 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Sum 2/0 11.27 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 670.9 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-09-21T12:30:17.571 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 670.9 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-09-21T12:30:17.571 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: 2024-09-21T12:30:17.571 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: ** Compaction Stats [default] ** 2024-09-21T12:30:17.572 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T12:30:17.572 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T12:30:17.572 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 670.9 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-09-21T12:30:17.572 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: 2024-09-21T12:30:17.572 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-21T12:30:17.572 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: 2024-09-21T12:30:17.572 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T12:30:17.572 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Flush(GB): cumulative 0.004, interval 0.004 2024-09-21T12:30:17.572 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T12:30:17.572 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T12:30:17.572 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T12:30:17.572 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T12:30:17.572 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Cumulative compaction: 0.00 GB write, 161.01 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T12:30:17.572 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Interval compaction: 0.00 GB write, 161.01 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T12:30:17.572 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T12:30:17.573 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Block cache BinnedLRUCache@0x55f6353a7610#2 capacity: 512.00 MB usage: 6.36 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.5e-05 secs_since: 0 2024-09-21T12:30:17.573 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.78 KB,0.000339746%) IndexBlock(1,4.58 KB,0.000873208%) Misc(1,0.00 KB,0%) 2024-09-21T12:30:17.573 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: 2024-09-21T12:30:17.573 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T12:30:17.573 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: starting mon.smithi148 rank 1 at public addrs [v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0] at bind addrs [v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi148 fsid 6263deca-7813-11ef-baf6-efdc52797490 2024-09-21T12:30:17.573 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: mon.smithi148@-1(???) e2 preinit fsid 6263deca-7813-11ef-baf6-efdc52797490 2024-09-21T12:30:17.573 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: mon.smithi148@-1(???).mds e7 new map 2024-09-21T12:30:17.573 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: mon.smithi148@-1(???).mds e7 print_map 2024-09-21T12:30:17.573 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: e7 2024-09-21T12:30:17.573 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-21T12:30:17.573 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T12:30:17.573 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T12:30:17.573 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: legacy client fscid: 1 2024-09-21T12:30:17.573 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: 2024-09-21T12:30:17.574 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Filesystem 'foofs' (1) 2024-09-21T12:30:17.574 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: fs_name foofs 2024-09-21T12:30:17.574 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: epoch 5 2024-09-21T12:30:17.574 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-21T12:30:17.574 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: created 2024-09-21T12:24:45.236210+0000 2024-09-21T12:30:17.574 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: modified 2024-09-21T12:24:51.265131+0000 2024-09-21T12:30:17.574 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: tableserver 0 2024-09-21T12:30:17.574 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: root 0 2024-09-21T12:30:17.574 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: session_timeout 60 2024-09-21T12:30:17.574 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: session_autoclose 300 2024-09-21T12:30:17.574 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: max_file_size 1099511627776 2024-09-21T12:30:17.574 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: max_xattr_size 65536 2024-09-21T12:30:17.574 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: required_client_features {} 2024-09-21T12:30:17.574 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: last_failure 0 2024-09-21T12:30:17.575 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: last_failure_osd_epoch 0 2024-09-21T12:30:17.575 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T12:30:17.575 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: max_mds 1 2024-09-21T12:30:17.575 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: in 0 2024-09-21T12:30:17.575 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: up {0=14494} 2024-09-21T12:30:17.575 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: failed 2024-09-21T12:30:17.575 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: damaged 2024-09-21T12:30:17.575 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: stopped 2024-09-21T12:30:17.575 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: data_pools [3] 2024-09-21T12:30:17.575 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: metadata_pool 2 2024-09-21T12:30:17.575 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: inline_data disabled 2024-09-21T12:30:17.575 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: balancer 2024-09-21T12:30:17.575 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: bal_rank_mask -1 2024-09-21T12:30:17.575 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: standby_count_wanted 1 2024-09-21T12:30:17.576 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: qdb_cluster leader: 0 members: 2024-09-21T12:30:17.576 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: [mds.foofs.smithi078.ytpzwh{0:14494} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.78:6834/169294426,v1:172.21.15.78:6835/169294426] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T12:30:17.576 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: 2024-09-21T12:30:17.576 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: 2024-09-21T12:30:17.576 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: Standby daemons: 2024-09-21T12:30:17.576 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: 2024-09-21T12:30:17.576 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout: [mds.foofs.smithi148.uukezr{-1:24267} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.148:6832/3237004392,v1:172.21.15.148:6833/3237004392] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T12:30:17.576 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: mon.smithi148@-1(???).osd e72 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-21T12:30:17.576 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: mon.smithi148@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T12:30:17.576 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: mon.smithi148@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T12:30:17.576 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: mon.smithi148@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-21T12:30:17.576 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 ceph-mon[49387]: mon.smithi148@-1(???).paxosservice(auth 1..30) refresh upgraded, format 0 -> 3 2024-09-21T12:30:17.576 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 bash[49369]: df8b0850913677c8d686548f8c3522af5680c9db06e6cd852dd2ed1ec5d0a17a 2024-09-21T12:30:17.576 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:17 smithi148 systemd[1]: Started Ceph mon.smithi148 for 6263deca-7813-11ef-baf6-efdc52797490. 2024-09-21T12:30:17.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 127871 10.57 MB/sec execute 175 sec latency 38.063 ms 2024-09-21T12:30:18.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 128139 10.55 MB/sec execute 176 sec latency 43.870 ms 2024-09-21T12:30:19.742 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:19 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mon metadata", "id": "smithi078"} : dispatch 2024-09-21T12:30:19.742 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:19 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mon metadata", "id": "smithi148"} : dispatch 2024-09-21T12:30:19.742 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:19 smithi078 ceph-mon[63866]: mon.smithi148 calling monitor election 2024-09-21T12:30:19.742 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:19 smithi078 ceph-mon[63866]: mon.smithi078 calling monitor election 2024-09-21T12:30:19.742 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:19 smithi078 ceph-mon[63866]: mon.smithi078 is new leader, mons smithi078,smithi148 in quorum (ranks 0,1) 2024-09-21T12:30:19.742 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:19 smithi078 ceph-mon[63866]: monmap epoch 3 2024-09-21T12:30:19.743 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:19 smithi078 ceph-mon[63866]: fsid 6263deca-7813-11ef-baf6-efdc52797490 2024-09-21T12:30:19.743 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:19 smithi078 ceph-mon[63866]: last_changed 2024-09-21T12:30:18.630545+0000 2024-09-21T12:30:19.743 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:19 smithi078 ceph-mon[63866]: created 2024-09-21T12:17:23.120662+0000 2024-09-21T12:30:19.743 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:19 smithi078 ceph-mon[63866]: min_mon_release 19 (squid) 2024-09-21T12:30:19.743 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:19 smithi078 ceph-mon[63866]: election_strategy: 1 2024-09-21T12:30:19.743 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:19 smithi078 ceph-mon[63866]: 0: [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon.smithi078 2024-09-21T12:30:19.743 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:19 smithi078 ceph-mon[63866]: 1: [v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0] mon.smithi148 2024-09-21T12:30:19.743 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:19 smithi078 ceph-mon[63866]: fsmap foofs:1 {0=foofs.smithi078.ytpzwh=up:active} 1 up:standby 2024-09-21T12:30:19.743 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:19 smithi078 ceph-mon[63866]: osdmap e72: 8 total, 8 up, 8 in 2024-09-21T12:30:19.743 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:19 smithi078 ceph-mon[63866]: mgrmap e42: smithi078.ntwyan(active, since 19s), standbys: smithi148.oyiqmx 2024-09-21T12:30:19.743 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:19 smithi078 ceph-mon[63866]: overall HEALTH_OK 2024-09-21T12:30:19.743 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:19 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:19.743 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:19 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:19.743 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:19 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:30:19.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 128362 10.50 MB/sec execute 177 sec latency 61.362 ms 2024-09-21T12:30:20.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:19 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mon metadata", "id": "smithi078"} : dispatch 2024-09-21T12:30:20.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:19 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mon metadata", "id": "smithi148"} : dispatch 2024-09-21T12:30:20.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:19 smithi148 ceph-mon[49387]: mon.smithi148 calling monitor election 2024-09-21T12:30:20.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:19 smithi148 ceph-mon[49387]: mon.smithi078 calling monitor election 2024-09-21T12:30:20.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:19 smithi148 ceph-mon[49387]: mon.smithi078 is new leader, mons smithi078,smithi148 in quorum (ranks 0,1) 2024-09-21T12:30:20.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:19 smithi148 ceph-mon[49387]: monmap epoch 3 2024-09-21T12:30:20.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:19 smithi148 ceph-mon[49387]: fsid 6263deca-7813-11ef-baf6-efdc52797490 2024-09-21T12:30:20.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:19 smithi148 ceph-mon[49387]: last_changed 2024-09-21T12:30:18.630545+0000 2024-09-21T12:30:20.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:19 smithi148 ceph-mon[49387]: created 2024-09-21T12:17:23.120662+0000 2024-09-21T12:30:20.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:19 smithi148 ceph-mon[49387]: min_mon_release 19 (squid) 2024-09-21T12:30:20.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:19 smithi148 ceph-mon[49387]: election_strategy: 1 2024-09-21T12:30:20.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:19 smithi148 ceph-mon[49387]: 0: [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon.smithi078 2024-09-21T12:30:20.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:19 smithi148 ceph-mon[49387]: 1: [v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0] mon.smithi148 2024-09-21T12:30:20.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:19 smithi148 ceph-mon[49387]: fsmap foofs:1 {0=foofs.smithi078.ytpzwh=up:active} 1 up:standby 2024-09-21T12:30:20.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:19 smithi148 ceph-mon[49387]: osdmap e72: 8 total, 8 up, 8 in 2024-09-21T12:30:20.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:19 smithi148 ceph-mon[49387]: mgrmap e42: smithi078.ntwyan(active, since 19s), standbys: smithi148.oyiqmx 2024-09-21T12:30:20.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:19 smithi148 ceph-mon[49387]: overall HEALTH_OK 2024-09-21T12:30:20.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:19 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:20.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:19 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:20.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:19 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:30:20.756 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:20 smithi078 ceph-mon[63866]: pgmap v13: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.4 MiB/s wr, 366 op/s 2024-09-21T12:30:20.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 128579 10.46 MB/sec execute 178 sec latency 83.181 ms 2024-09-21T12:30:20.961 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:20 smithi148 ceph-mon[49387]: pgmap v13: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.4 MiB/s wr, 366 op/s 2024-09-21T12:30:21.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 128871 10.44 MB/sec execute 179 sec latency 54.925 ms 2024-09-21T12:30:22.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 129240 10.42 MB/sec execute 180 sec latency 60.472 ms 2024-09-21T12:30:22.801 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:22 smithi078 ceph-mon[63866]: pgmap v14: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.9 MiB/s wr, 573 op/s 2024-09-21T12:30:22.802 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:22 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:22.802 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:22 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:23.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:22 smithi148 ceph-mon[49387]: pgmap v14: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.9 MiB/s wr, 573 op/s 2024-09-21T12:30:23.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:22 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:23.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:22 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:23.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 129573 10.40 MB/sec execute 181 sec latency 55.008 ms 2024-09-21T12:30:24.340 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:24 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:24.340 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:24 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:24.340 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:24 smithi078 ceph-mon[63866]: pgmap v15: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 426 op/s 2024-09-21T12:30:24.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:24 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:24.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:24 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:24.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:24 smithi148 ceph-mon[49387]: pgmap v15: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 426 op/s 2024-09-21T12:30:24.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 129985 10.40 MB/sec execute 182 sec latency 71.057 ms 2024-09-21T12:30:25.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 130488 10.43 MB/sec execute 183 sec latency 75.816 ms 2024-09-21T12:30:26.627 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:26 smithi078 ceph-mon[63866]: pgmap v16: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 426 op/s 2024-09-21T12:30:26.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 131152 10.52 MB/sec execute 184 sec latency 26.600 ms 2024-09-21T12:30:26.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:26 smithi148 ceph-mon[49387]: pgmap v16: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 426 op/s 2024-09-21T12:30:27.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 131456 10.52 MB/sec execute 185 sec latency 41.293 ms 2024-09-21T12:30:28.665 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:28 smithi078 ceph-mon[63866]: pgmap v17: 97 pgs: 97 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.1 MiB/s wr, 638 op/s 2024-09-21T12:30:28.666 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:28 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:28.666 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:28 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:28.666 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:28 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:28.666 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:28 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:30:28.666 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:28 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:28.666 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:28 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:28.666 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:28 smithi078 ceph-mon[63866]: Reconfiguring mon.smithi078 (monmap changed)... 2024-09-21T12:30:28.666 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:28 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T12:30:28.666 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:28 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T12:30:28.666 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:28 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:28.666 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:28 smithi078 ceph-mon[63866]: Reconfiguring daemon mon.smithi078 on smithi078 2024-09-21T12:30:28.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 131707 10.49 MB/sec execute 186 sec latency 48.321 ms 2024-09-21T12:30:29.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:28 smithi148 ceph-mon[49387]: pgmap v17: 97 pgs: 97 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.1 MiB/s wr, 638 op/s 2024-09-21T12:30:29.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:28 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:29.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:28 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:29.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:28 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:29.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:28 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:30:29.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:28 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:29.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:28 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:29.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:28 smithi148 ceph-mon[49387]: Reconfiguring mon.smithi078 (monmap changed)... 2024-09-21T12:30:29.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:28 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T12:30:29.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:28 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T12:30:29.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:28 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:29.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:28 smithi148 ceph-mon[49387]: Reconfiguring daemon mon.smithi078 on smithi078 2024-09-21T12:30:29.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 131911 10.45 MB/sec execute 187 sec latency 62.112 ms 2024-09-21T12:30:30.696 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:30 smithi078 ceph-mon[63866]: pgmap v18: 97 pgs: 97 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.2 MiB/s wr, 419 op/s 2024-09-21T12:30:30.697 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:30:30.697 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:30.697 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:30.697 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi078.ntwyan", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T12:30:30.697 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T12:30:30.697 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:30.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 132191 10.42 MB/sec execute 188 sec latency 55.869 ms 2024-09-21T12:30:31.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:30 smithi148 ceph-mon[49387]: pgmap v18: 97 pgs: 97 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.2 MiB/s wr, 419 op/s 2024-09-21T12:30:31.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:30:31.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:31.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:31.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi078.ntwyan", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T12:30:31.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T12:30:31.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:31.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:31 smithi078 ceph-mon[63866]: Reconfiguring mgr.smithi078.ntwyan (monmap changed)... 2024-09-21T12:30:31.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:31 smithi078 ceph-mon[63866]: Reconfiguring daemon mgr.smithi078.ntwyan on smithi078 2024-09-21T12:30:31.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 132545 10.41 MB/sec execute 189 sec latency 80.961 ms 2024-09-21T12:30:32.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:31 smithi148 ceph-mon[49387]: Reconfiguring mgr.smithi078.ntwyan (monmap changed)... 2024-09-21T12:30:32.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:31 smithi148 ceph-mon[49387]: Reconfiguring daemon mgr.smithi078.ntwyan on smithi078 2024-09-21T12:30:32.749 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:32 smithi078 ceph-mon[63866]: pgmap v19: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.9 MiB/s wr, 647 op/s 2024-09-21T12:30:32.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 132861 10.38 MB/sec execute 190 sec latency 81.640 ms 2024-09-21T12:30:33.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:32 smithi148 ceph-mon[49387]: pgmap v19: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.9 MiB/s wr, 647 op/s 2024-09-21T12:30:33.772 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:33 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:33.773 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:33 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:33.773 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:33 smithi078 ceph-mon[63866]: Reconfiguring crash.smithi078 (monmap changed)... 2024-09-21T12:30:33.773 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:33 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T12:30:33.773 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:33 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:33.773 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:33 smithi078 ceph-mon[63866]: Reconfiguring daemon crash.smithi078 on smithi078 2024-09-21T12:30:33.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 133229 10.36 MB/sec execute 191 sec latency 57.513 ms 2024-09-21T12:30:34.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:33 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:34.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:33 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:34.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:33 smithi148 ceph-mon[49387]: Reconfiguring crash.smithi078 (monmap changed)... 2024-09-21T12:30:34.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:33 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T12:30:34.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:33 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:34.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:33 smithi148 ceph-mon[49387]: Reconfiguring daemon crash.smithi078 on smithi078 2024-09-21T12:30:34.790 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:34 smithi078 ceph-mon[63866]: pgmap v20: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.5 MiB/s wr, 440 op/s 2024-09-21T12:30:34.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 133615 10.36 MB/sec execute 192 sec latency 40.558 ms 2024-09-21T12:30:34.951 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:34 smithi148 ceph-mon[49387]: pgmap v20: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.5 MiB/s wr, 440 op/s 2024-09-21T12:30:35.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 133840 10.33 MB/sec execute 193 sec latency 60.015 ms 2024-09-21T12:30:36.207 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:35 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:36.207 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:35 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:36.207 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:35 smithi078 ceph-mon[63866]: Reconfiguring osd.0 (monmap changed)... 2024-09-21T12:30:36.207 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:35 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T12:30:36.207 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:35 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:36.207 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:35 smithi078 ceph-mon[63866]: Reconfiguring daemon osd.0 on smithi078 2024-09-21T12:30:36.208 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:35 smithi078 ceph-mon[63866]: pgmap v21: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.5 MiB/s wr, 440 op/s 2024-09-21T12:30:36.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:35 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:36.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:35 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:36.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:35 smithi148 ceph-mon[49387]: Reconfiguring osd.0 (monmap changed)... 2024-09-21T12:30:36.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:35 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T12:30:36.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:35 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:36.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:35 smithi148 ceph-mon[49387]: Reconfiguring daemon osd.0 on smithi078 2024-09-21T12:30:36.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:35 smithi148 ceph-mon[49387]: pgmap v21: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.5 MiB/s wr, 440 op/s 2024-09-21T12:30:36.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 134344 10.39 MB/sec execute 194 sec latency 34.853 ms 2024-09-21T12:30:37.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 134687 10.42 MB/sec execute 195 sec latency 50.077 ms 2024-09-21T12:30:38.762 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:38 smithi078 ceph-mon[63866]: pgmap v22: 97 pgs: 97 active+clean; 330 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.1 MiB/s wr, 642 op/s 2024-09-21T12:30:38.762 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:38 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:38.762 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:38 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:38.762 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:38 smithi078 ceph-mon[63866]: Reconfiguring osd.1 (monmap changed)... 2024-09-21T12:30:38.762 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:38 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T12:30:38.762 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:38 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:38.762 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:38 smithi078 ceph-mon[63866]: Reconfiguring daemon osd.1 on smithi078 2024-09-21T12:30:38.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 134974 10.42 MB/sec execute 196 sec latency 42.499 ms 2024-09-21T12:30:39.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:38 smithi148 ceph-mon[49387]: pgmap v22: 97 pgs: 97 active+clean; 330 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.1 MiB/s wr, 642 op/s 2024-09-21T12:30:39.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:38 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:39.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:38 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:39.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:38 smithi148 ceph-mon[49387]: Reconfiguring osd.1 (monmap changed)... 2024-09-21T12:30:39.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:38 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T12:30:39.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:38 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:39.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:38 smithi148 ceph-mon[49387]: Reconfiguring daemon osd.1 on smithi078 2024-09-21T12:30:39.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 135262 10.40 MB/sec execute 197 sec latency 33.766 ms 2024-09-21T12:30:40.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 135640 10.37 MB/sec execute 198 sec latency 58.889 ms 2024-09-21T12:30:40.816 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:40 smithi078 ceph-mon[63866]: pgmap v23: 97 pgs: 97 active+clean; 330 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.4 MiB/s wr, 430 op/s 2024-09-21T12:30:40.817 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:40 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:40.817 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:40 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:40.817 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:40 smithi078 ceph-mon[63866]: Reconfiguring osd.2 (monmap changed)... 2024-09-21T12:30:40.817 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:40 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T12:30:40.817 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:40 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:40.817 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:40 smithi078 ceph-mon[63866]: Reconfiguring daemon osd.2 on smithi078 2024-09-21T12:30:41.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:40 smithi148 ceph-mon[49387]: pgmap v23: 97 pgs: 97 active+clean; 330 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.4 MiB/s wr, 430 op/s 2024-09-21T12:30:41.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:40 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:41.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:40 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:41.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:40 smithi148 ceph-mon[49387]: Reconfiguring osd.2 (monmap changed)... 2024-09-21T12:30:41.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:40 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T12:30:41.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:40 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:41.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:40 smithi148 ceph-mon[49387]: Reconfiguring daemon osd.2 on smithi078 2024-09-21T12:30:41.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 136026 10.35 MB/sec execute 199 sec latency 32.201 ms 2024-09-21T12:30:42.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 136528 10.37 MB/sec execute 200 sec latency 39.871 ms 2024-09-21T12:30:42.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:42 smithi078 ceph-mon[63866]: pgmap v24: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.9 MiB/s wr, 639 op/s 2024-09-21T12:30:42.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:42 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:42.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:42 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:42.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:42 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T12:30:42.852 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:42 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:43.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:42 smithi148 ceph-mon[49387]: pgmap v24: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.9 MiB/s wr, 639 op/s 2024-09-21T12:30:43.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:42 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:43.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:42 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:43.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:42 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T12:30:43.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:42 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:43.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 136990 10.35 MB/sec execute 201 sec latency 64.632 ms 2024-09-21T12:30:43.879 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:43 smithi078 ceph-mon[63866]: Reconfiguring osd.3 (monmap changed)... 2024-09-21T12:30:43.879 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:43 smithi078 ceph-mon[63866]: Reconfiguring daemon osd.3 on smithi078 2024-09-21T12:30:44.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:43 smithi148 ceph-mon[49387]: Reconfiguring osd.3 (monmap changed)... 2024-09-21T12:30:44.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:43 smithi148 ceph-mon[49387]: Reconfiguring daemon osd.3 on smithi078 2024-09-21T12:30:44.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 137445 10.37 MB/sec execute 202 sec latency 53.863 ms 2024-09-21T12:30:44.888 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:44 smithi078 ceph-mon[63866]: pgmap v25: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.3 MiB/s wr, 411 op/s 2024-09-21T12:30:44.888 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:44 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:44.888 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:44 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:44.888 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:44 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.ytpzwh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T12:30:44.888 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:44 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:45.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:44 smithi148 ceph-mon[49387]: pgmap v25: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.3 MiB/s wr, 411 op/s 2024-09-21T12:30:45.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:44 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:45.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:44 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:45.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:44 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.ytpzwh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T12:30:45.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:44 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:45.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 137715 10.37 MB/sec execute 203 sec latency 51.010 ms 2024-09-21T12:30:45.911 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:45 smithi078 ceph-mon[63866]: Reconfiguring mds.foofs.smithi078.ytpzwh (monmap changed)... 2024-09-21T12:30:45.911 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:45 smithi078 ceph-mon[63866]: Reconfiguring daemon mds.foofs.smithi078.ytpzwh on smithi078 2024-09-21T12:30:45.912 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:30:46.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:45 smithi148 ceph-mon[49387]: Reconfiguring mds.foofs.smithi078.ytpzwh (monmap changed)... 2024-09-21T12:30:46.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:45 smithi148 ceph-mon[49387]: Reconfiguring daemon mds.foofs.smithi078.ytpzwh on smithi078 2024-09-21T12:30:46.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:30:46.800 INFO:teuthology.orchestra.run.smithi078.stdout: 5 138168 10.41 MB/sec execute 204 sec latency 42.191 ms 2024-09-21T12:30:46.804 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-21T12:30:46.934 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:46 smithi078 ceph-mon[63866]: pgmap v26: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.3 MiB/s wr, 411 op/s 2024-09-21T12:30:47.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:46 smithi148 ceph-mon[49387]: pgmap v26: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.3 MiB/s wr, 411 op/s 2024-09-21T12:30:47.800 INFO:teuthology.orchestra.run.smithi078.stdout: 5 138709 10.45 MB/sec execute 205 sec latency 43.602 ms 2024-09-21T12:30:47.944 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:47 smithi078 ceph-mon[63866]: from='client.34150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:30:47.944 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:47 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:47.944 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:47 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:47.944 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:47 smithi078 ceph-mon[63866]: Reconfiguring crash.smithi148 (monmap changed)... 2024-09-21T12:30:47.944 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:47 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi148", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T12:30:47.944 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:47 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:47.944 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:47 smithi078 ceph-mon[63866]: Reconfiguring daemon crash.smithi148 on smithi148 2024-09-21T12:30:47.944 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:47 smithi078 ceph-mon[63866]: from='client.34154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:30:47.945 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:47 smithi078 ceph-mon[63866]: pgmap v27: 97 pgs: 97 active+clean; 334 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.5 MiB/s wr, 666 op/s 2024-09-21T12:30:47.984 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T12:30:47.984 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (3m) 38s ago 11m 20.7M - 0.27.0 11f11916f8cd e6e67ab07dec 2024-09-21T12:30:47.984 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (11m) 38s ago 11m 6400k - 17.2.0 e1d6a67b021e 4a209e5da5ce 2024-09-21T12:30:47.984 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi148 smithi148 running (11m) 25s ago 11m 6405k - 17.2.0 e1d6a67b021e 0fa5e612307f 2024-09-21T12:30:47.985 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (2m) 38s ago 11m 53.3M - dad864ee21e9 2a140c36091b 2024-09-21T12:30:47.985 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.ytpzwh smithi078 running (5m) 38s ago 5m 37.4M - 17.2.0 e1d6a67b021e 5e9b109502a2 2024-09-21T12:30:47.985 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi148.uukezr smithi148 running (5m) 25s ago 5m 13.6M - 17.2.0 e1d6a67b021e b7e8aa0f3a6c 2024-09-21T12:30:47.985 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.ntwyan smithi078 *:8443,9283,8765 running (2m) 38s ago 13m 488M - 19.3.0-5104-g44178494 b415816db8b4 a61baedebf43 2024-09-21T12:30:47.985 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi148.oyiqmx smithi148 *:8443,9283,8765 running (74s) 25s ago 11m 452M - 19.3.0-5104-g44178494 b415816db8b4 0255535c6fdb 2024-09-21T12:30:47.985 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (56s) 38s ago 13m 34.4M 2048M 19.3.0-5104-g44178494 b415816db8b4 22b3097efbd8 2024-09-21T12:30:47.985 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi148 smithi148 running (31s) 25s ago 11m 24.5M 2048M 19.3.0-5104-g44178494 b415816db8b4 df8b08509136 2024-09-21T12:30:47.985 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.ydbhru smithi078 *:2049 running (5m) 38s ago 5m 262M - 4.0 e1d6a67b021e 7e5b0274ac38 2024-09-21T12:30:47.985 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi148.einuzt smithi148 *:2049 running (5m) 25s ago 5m 86.7M - 4.0 e1d6a67b021e 76a8b0eabede 2024-09-21T12:30:47.985 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (2m) 38s ago 11m 11.5M - 1.7.0 72c9c2088986 301adb4e9e22 2024-09-21T12:30:47.985 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi148 smithi148 *:9100 running (2m) 25s ago 11m 12.1M - 1.7.0 72c9c2088986 0cceff4bf342 2024-09-21T12:30:47.985 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (10m) 38s ago 10m 453M 1936M 17.2.0 e1d6a67b021e d4d5e9977070 2024-09-21T12:30:47.985 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (9m) 38s ago 9m 402M 1936M 17.2.0 e1d6a67b021e b9a4b952233b 2024-09-21T12:30:47.985 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (9m) 38s ago 9m 308M 1936M 17.2.0 e1d6a67b021e 829fb8c2d827 2024-09-21T12:30:47.986 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (9m) 38s ago 9m 416M 1936M 17.2.0 e1d6a67b021e ef942c8556d0 2024-09-21T12:30:47.986 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi148 running (8m) 25s ago 8m 455M 2704M 17.2.0 e1d6a67b021e 62eefdd572c0 2024-09-21T12:30:47.986 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi148 running (8m) 25s ago 8m 497M 2704M 17.2.0 e1d6a67b021e 7b44be78baca 2024-09-21T12:30:47.986 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi148 running (7m) 25s ago 7m 419M 2704M 17.2.0 e1d6a67b021e 3d23057910ea 2024-09-21T12:30:47.986 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi148 running (7m) 25s ago 7m 534M 2704M 17.2.0 e1d6a67b021e f9e4db9d35fa 2024-09-21T12:30:47.986 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (80s) 38s ago 10m 47.4M - 2.51.0 1d3b7f56885b face2d4cdd1b 2024-09-21T12:30:48.256 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:47 smithi148 ceph-mon[49387]: from='client.34150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:30:48.256 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:47 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:48.256 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:47 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:48.256 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:47 smithi148 ceph-mon[49387]: Reconfiguring crash.smithi148 (monmap changed)... 2024-09-21T12:30:48.256 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:47 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi148", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T12:30:48.256 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:47 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:48.256 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:47 smithi148 ceph-mon[49387]: Reconfiguring daemon crash.smithi148 on smithi148 2024-09-21T12:30:48.256 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:47 smithi148 ceph-mon[49387]: from='client.34154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:30:48.256 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:47 smithi148 ceph-mon[49387]: pgmap v27: 97 pgs: 97 active+clean; 334 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.5 MiB/s wr, 666 op/s 2024-09-21T12:30:48.702 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:30:48.702 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-21T12:30:48.702 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:30:48.702 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:30:48.703 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-21T12:30:48.703 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:30:48.703 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:30:48.703 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-21T12:30:48.703 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T12:30:48.703 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:30:48.703 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-21T12:30:48.703 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:30:48.703 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:30:48.703 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-21T12:30:48.703 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-21T12:30:48.703 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 4 2024-09-21T12:30:48.703 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T12:30:48.703 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:30:48.799 INFO:teuthology.orchestra.run.smithi078.stdout: 5 139209 10.44 MB/sec execute 206 sec latency 28.206 ms 2024-09-21T12:30:49.356 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:30:49.356 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5", 2024-09-21T12:30:49.356 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-21T12:30:49.356 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T12:30:49.356 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-21T12:30:49.356 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-09-21T12:30:49.356 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr" 2024-09-21T12:30:49.356 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-21T12:30:49.356 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "4/23 daemons upgraded", 2024-09-21T12:30:49.357 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading mon daemons", 2024-09-21T12:30:49.357 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-21T12:30:49.357 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:30:49.452 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:49 smithi078 ceph-mon[63866]: from='client.44105 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:30:49.452 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:49 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:49.452 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:49 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:49.452 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:49 smithi078 ceph-mon[63866]: Reconfiguring mgr.smithi148.oyiqmx (monmap changed)... 2024-09-21T12:30:49.452 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:49 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi148.oyiqmx", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T12:30:49.452 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:49 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T12:30:49.452 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:49 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:49.452 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:49 smithi078 ceph-mon[63866]: Reconfiguring daemon mgr.smithi148.oyiqmx on smithi148 2024-09-21T12:30:49.452 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:49 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/3625985707' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:30:49.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:49 smithi148 ceph-mon[49387]: from='client.44105 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:30:49.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:49 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:49.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:49 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:49.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:49 smithi148 ceph-mon[49387]: Reconfiguring mgr.smithi148.oyiqmx (monmap changed)... 2024-09-21T12:30:49.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:49 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi148.oyiqmx", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T12:30:49.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:49 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T12:30:49.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:49 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:49.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:49 smithi148 ceph-mon[49387]: Reconfiguring daemon mgr.smithi148.oyiqmx on smithi148 2024-09-21T12:30:49.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:49 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/3625985707' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:30:49.800 INFO:teuthology.orchestra.run.smithi078.stdout: 5 139817 10.44 MB/sec execute 207 sec latency 22.760 ms 2024-09-21T12:30:50.090 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-21T12:30:50.462 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:50 smithi078 ceph-mon[63866]: from='client.44113 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:30:50.462 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:50 smithi078 ceph-mon[63866]: pgmap v28: 97 pgs: 97 active+clean; 334 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.8 MiB/s wr, 463 op/s 2024-09-21T12:30:50.462 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:50 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:50.462 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:50 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:50.462 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:50 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T12:30:50.462 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:50 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T12:30:50.462 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:50 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:50.462 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:50 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/984015986' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:30:50.662 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:50 smithi148 ceph-mon[49387]: from='client.44113 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:30:50.662 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:50 smithi148 ceph-mon[49387]: pgmap v28: 97 pgs: 97 active+clean; 334 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.8 MiB/s wr, 463 op/s 2024-09-21T12:30:50.662 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:50 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:50.662 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:50 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:50.662 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:50 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T12:30:50.662 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:50 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T12:30:50.663 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:50 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:50.663 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:50 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/984015986' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:30:50.800 INFO:teuthology.orchestra.run.smithi078.stdout: 5 140491 10.47 MB/sec execute 208 sec latency 56.237 ms 2024-09-21T12:30:51.489 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:51 smithi078 ceph-mon[63866]: Reconfiguring mon.smithi148 (monmap changed)... 2024-09-21T12:30:51.489 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:51 smithi078 ceph-mon[63866]: Reconfiguring daemon mon.smithi148 on smithi148 2024-09-21T12:30:51.800 INFO:teuthology.orchestra.run.smithi078.stdout: 5 140892 10.46 MB/sec execute 209 sec latency 46.939 ms 2024-09-21T12:30:51.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:51 smithi148 ceph-mon[49387]: Reconfiguring mon.smithi148 (monmap changed)... 2024-09-21T12:30:51.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:51 smithi148 ceph-mon[49387]: Reconfiguring daemon mon.smithi148 on smithi148 2024-09-21T12:30:52.769 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:52 smithi078 ceph-mon[63866]: pgmap v29: 97 pgs: 97 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 8.1 MiB/s wr, 783 op/s 2024-09-21T12:30:52.769 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:52.769 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:52.769 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:52 smithi078 ceph-mon[63866]: Reconfiguring osd.4 (monmap changed)... 2024-09-21T12:30:52.769 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T12:30:52.769 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:52.769 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:52 smithi078 ceph-mon[63866]: Reconfiguring daemon osd.4 on smithi148 2024-09-21T12:30:52.800 INFO:teuthology.orchestra.run.smithi078.stdout: 5 141229 10.48 MB/sec execute 210 sec latency 60.081 ms 2024-09-21T12:30:52.859 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:52 smithi148 ceph-mon[49387]: pgmap v29: 97 pgs: 97 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 8.1 MiB/s wr, 783 op/s 2024-09-21T12:30:52.859 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:52.859 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:52.859 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:52 smithi148 ceph-mon[49387]: Reconfiguring osd.4 (monmap changed)... 2024-09-21T12:30:52.859 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T12:30:52.859 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:52.859 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:52 smithi148 ceph-mon[49387]: Reconfiguring daemon osd.4 on smithi148 2024-09-21T12:30:53.800 INFO:teuthology.orchestra.run.smithi078.stdout: 5 141769 10.53 MB/sec execute 211 sec latency 37.685 ms 2024-09-21T12:30:54.289 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:54 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:54.289 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:54 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:54.289 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:54 smithi078 ceph-mon[63866]: Reconfiguring osd.5 (monmap changed)... 2024-09-21T12:30:54.289 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:54 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T12:30:54.289 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:54 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:54.289 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:54 smithi078 ceph-mon[63866]: Reconfiguring daemon osd.5 on smithi148 2024-09-21T12:30:54.289 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:54 smithi078 ceph-mon[63866]: pgmap v30: 97 pgs: 97 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.5 MiB/s wr, 575 op/s 2024-09-21T12:30:54.340 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:54 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:54.340 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:54 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:54.340 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:54 smithi148 ceph-mon[49387]: Reconfiguring osd.5 (monmap changed)... 2024-09-21T12:30:54.341 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:54 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T12:30:54.341 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:54 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:54.341 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:54 smithi148 ceph-mon[49387]: Reconfiguring daemon osd.5 on smithi148 2024-09-21T12:30:54.341 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:54 smithi148 ceph-mon[49387]: pgmap v30: 97 pgs: 97 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.5 MiB/s wr, 575 op/s 2024-09-21T12:30:54.800 INFO:teuthology.orchestra.run.smithi078.stdout: 5 142239 10.55 MB/sec execute 212 sec latency 28.477 ms 2024-09-21T12:30:55.800 INFO:teuthology.orchestra.run.smithi078.stdout: 5 142790 10.55 MB/sec execute 213 sec latency 40.920 ms 2024-09-21T12:30:56.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:56 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:56.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:56 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:56.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:56 smithi078 ceph-mon[63866]: Reconfiguring osd.6 (monmap changed)... 2024-09-21T12:30:56.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:56 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T12:30:56.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:56 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:56.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:56 smithi078 ceph-mon[63866]: Reconfiguring daemon osd.6 on smithi148 2024-09-21T12:30:56.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:56 smithi078 ceph-mon[63866]: pgmap v31: 97 pgs: 97 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.5 MiB/s wr, 575 op/s 2024-09-21T12:30:56.712 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:56 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:56.712 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:56 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:56.712 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:56 smithi148 ceph-mon[49387]: Reconfiguring osd.6 (monmap changed)... 2024-09-21T12:30:56.712 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:56 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T12:30:56.712 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:56 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:56.712 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:56 smithi148 ceph-mon[49387]: Reconfiguring daemon osd.6 on smithi148 2024-09-21T12:30:56.712 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:56 smithi148 ceph-mon[49387]: pgmap v31: 97 pgs: 97 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.5 MiB/s wr, 575 op/s 2024-09-21T12:30:56.800 INFO:teuthology.orchestra.run.smithi078.stdout: 5 143239 10.53 MB/sec execute 214 sec latency 68.396 ms 2024-09-21T12:30:57.800 INFO:teuthology.orchestra.run.smithi078.stdout: 5 143533 10.52 MB/sec execute 215 sec latency 51.761 ms 2024-09-21T12:30:57.836 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:57 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:57.836 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:57 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:57.836 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:57 smithi078 ceph-mon[63866]: Reconfiguring osd.7 (monmap changed)... 2024-09-21T12:30:57.836 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:57 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T12:30:57.836 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:57 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:57.836 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:57 smithi078 ceph-mon[63866]: Reconfiguring daemon osd.7 on smithi148 2024-09-21T12:30:57.836 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:57 smithi078 ceph-mon[63866]: pgmap v32: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.7 MiB/s wr, 846 op/s 2024-09-21T12:30:58.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:57 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:58.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:57 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:58.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:57 smithi148 ceph-mon[49387]: Reconfiguring osd.7 (monmap changed)... 2024-09-21T12:30:58.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:57 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T12:30:58.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:57 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:58.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:57 smithi148 ceph-mon[49387]: Reconfiguring daemon osd.7 on smithi148 2024-09-21T12:30:58.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:57 smithi148 ceph-mon[49387]: pgmap v32: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.7 MiB/s wr, 846 op/s 2024-09-21T12:30:58.800 INFO:teuthology.orchestra.run.smithi078.stdout: 5 144028 10.53 MB/sec execute 216 sec latency 47.242 ms 2024-09-21T12:30:59.672 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:59 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:59.672 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:59 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:30:59.672 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:59 smithi078 ceph-mon[63866]: Reconfiguring mds.foofs.smithi148.uukezr (monmap changed)... 2024-09-21T12:30:59.672 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:59 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi148.uukezr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T12:30:59.672 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:59 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:30:59.672 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:30:59 smithi078 ceph-mon[63866]: Reconfiguring daemon mds.foofs.smithi148.uukezr on smithi148 2024-09-21T12:30:59.800 INFO:teuthology.orchestra.run.smithi078.stdout: 5 144434 10.51 MB/sec execute 217 sec latency 62.444 ms 2024-09-21T12:31:00.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:59 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:00.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:59 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:00.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:59 smithi148 ceph-mon[49387]: Reconfiguring mds.foofs.smithi148.uukezr (monmap changed)... 2024-09-21T12:31:00.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:59 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi148.uukezr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T12:31:00.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:59 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:31:00.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:30:59 smithi148 ceph-mon[49387]: Reconfiguring daemon mds.foofs.smithi148.uukezr on smithi148 2024-09-21T12:31:00.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:00 smithi078 ceph-mon[63866]: pgmap v33: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.4 MiB/s wr, 591 op/s 2024-09-21T12:31:00.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:31:00.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:00.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:00.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:31:00.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:31:00.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:31:00.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:00.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi078"} : dispatch 2024-09-21T12:31:00.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi078"}]': finished 2024-09-21T12:31:00.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi148"} : dispatch 2024-09-21T12:31:00.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi148"}]': finished 2024-09-21T12:31:00.802 INFO:teuthology.orchestra.run.smithi078.stdout: 5 144816 10.54 MB/sec execute 218 sec latency 45.652 ms 2024-09-21T12:31:01.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:00 smithi148 ceph-mon[49387]: pgmap v33: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.4 MiB/s wr, 591 op/s 2024-09-21T12:31:01.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:31:01.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:01.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:01.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:31:01.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:31:01.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:31:01.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:01.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi078"} : dispatch 2024-09-21T12:31:01.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi078"}]': finished 2024-09-21T12:31:01.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi148"} : dispatch 2024-09-21T12:31:01.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi148"}]': finished 2024-09-21T12:31:01.755 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:01 smithi078 ceph-mon[63866]: Upgrade: Setting container_image for all mon 2024-09-21T12:31:01.800 INFO:teuthology.orchestra.run.smithi078.stdout: 5 145213 10.57 MB/sec execute 219 sec latency 85.957 ms 2024-09-21T12:31:02.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:01 smithi148 ceph-mon[49387]: Upgrade: Setting container_image for all mon 2024-09-21T12:31:02.800 INFO:teuthology.orchestra.run.smithi078.stdout: 5 145605 10.57 MB/sec execute 220 sec latency 40.619 ms 2024-09-21T12:31:02.828 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:02 smithi078 ceph-mon[63866]: pgmap v34: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.7 MiB/s wr, 847 op/s 2024-09-21T12:31:03.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:02 smithi148 ceph-mon[49387]: pgmap v34: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.7 MiB/s wr, 847 op/s 2024-09-21T12:31:03.800 INFO:teuthology.orchestra.run.smithi078.stdout: 5 145902 10.57 MB/sec execute 221 sec latency 66.650 ms 2024-09-21T12:31:03.865 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:03 smithi078 ceph-mon[63866]: Upgrade: Updating crash.smithi078 (1/2) 2024-09-21T12:31:03.865 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:03 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:03.865 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:03 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T12:31:03.865 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:03 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:31:03.865 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:03 smithi078 ceph-mon[63866]: Deploying daemon crash.smithi078 on smithi078 2024-09-21T12:31:04.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:03 smithi148 ceph-mon[49387]: Upgrade: Updating crash.smithi078 (1/2) 2024-09-21T12:31:04.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:03 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:04.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:03 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T12:31:04.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:03 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:31:04.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:03 smithi148 ceph-mon[49387]: Deploying daemon crash.smithi078 on smithi078 2024-09-21T12:31:04.800 INFO:teuthology.orchestra.run.smithi078.stdout: 5 146171 10.53 MB/sec execute 222 sec latency 46.507 ms 2024-09-21T12:31:04.878 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:04 smithi078 ceph-mon[63866]: pgmap v35: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.4 MiB/s wr, 527 op/s 2024-09-21T12:31:04.952 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:04 smithi148 ceph-mon[49387]: pgmap v35: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.4 MiB/s wr, 527 op/s 2024-09-21T12:31:05.800 INFO:teuthology.orchestra.run.smithi078.stdout: 5 146463 10.51 MB/sec execute 223 sec latency 56.263 ms 2024-09-21T12:31:06.724 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:06 smithi078 ceph-mon[63866]: pgmap v36: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.4 MiB/s wr, 527 op/s 2024-09-21T12:31:06.800 INFO:teuthology.orchestra.run.smithi078.stdout: 5 146781 10.49 MB/sec execute 224 sec latency 43.222 ms 2024-09-21T12:31:07.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:06 smithi148 ceph-mon[49387]: pgmap v36: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.4 MiB/s wr, 527 op/s 2024-09-21T12:31:07.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 147049 10.48 MB/sec execute 225 sec latency 43.078 ms 2024-09-21T12:31:08.775 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:08 smithi078 ceph-mon[63866]: pgmap v37: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 729 op/s 2024-09-21T12:31:08.775 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:08 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:08.775 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:08 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:08.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 147576 10.49 MB/sec execute 226 sec latency 32.417 ms 2024-09-21T12:31:08.960 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:08 smithi148 ceph-mon[49387]: pgmap v37: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 729 op/s 2024-09-21T12:31:08.960 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:08 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:08.960 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:08 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:09.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 147933 10.46 MB/sec execute 227 sec latency 48.744 ms 2024-09-21T12:31:10.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 148275 10.47 MB/sec execute 228 sec latency 54.300 ms 2024-09-21T12:31:10.862 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:10 smithi078 ceph-mon[63866]: pgmap v38: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.6 MiB/s wr, 457 op/s 2024-09-21T12:31:10.863 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:10 smithi078 ceph-mon[63866]: Upgrade: Updating crash.smithi148 (2/2) 2024-09-21T12:31:10.863 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:10 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:10.863 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:10 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi148", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T12:31:10.863 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:10 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:31:10.863 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:10 smithi078 ceph-mon[63866]: Deploying daemon crash.smithi148 on smithi148 2024-09-21T12:31:11.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:10 smithi148 ceph-mon[49387]: pgmap v38: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.6 MiB/s wr, 457 op/s 2024-09-21T12:31:11.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:10 smithi148 ceph-mon[49387]: Upgrade: Updating crash.smithi148 (2/2) 2024-09-21T12:31:11.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:10 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:11.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:10 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi148", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-21T12:31:11.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:10 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:31:11.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:10 smithi148 ceph-mon[49387]: Deploying daemon crash.smithi148 on smithi148 2024-09-21T12:31:11.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 148613 10.50 MB/sec execute 229 sec latency 37.785 ms 2024-09-21T12:31:12.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 148922 10.49 MB/sec execute 230 sec latency 72.474 ms 2024-09-21T12:31:12.900 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:12 smithi078 ceph-mon[63866]: pgmap v39: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.5 MiB/s wr, 674 op/s 2024-09-21T12:31:13.003 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:12 smithi148 ceph-mon[49387]: pgmap v39: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.5 MiB/s wr, 674 op/s 2024-09-21T12:31:13.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 149267 10.50 MB/sec execute 231 sec latency 47.474 ms 2024-09-21T12:31:14.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 149538 10.48 MB/sec execute 232 sec latency 69.556 ms 2024-09-21T12:31:15.027 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:14 smithi078 ceph-mon[63866]: pgmap v40: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 418 op/s 2024-09-21T12:31:15.027 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:14 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:15.027 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:14 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:15.027 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:14 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:31:15.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:14 smithi148 ceph-mon[49387]: pgmap v40: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 418 op/s 2024-09-21T12:31:15.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:14 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:15.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:14 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:15.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:14 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:31:15.800 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:15 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:31:15.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 149793 10.46 MB/sec execute 233 sec latency 77.936 ms 2024-09-21T12:31:16.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:15 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:31:16.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 150253 10.44 MB/sec execute 234 sec latency 73.676 ms 2024-09-21T12:31:16.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:16 smithi078 ceph-mon[63866]: pgmap v41: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 418 op/s 2024-09-21T12:31:17.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:16 smithi148 ceph-mon[49387]: pgmap v41: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 418 op/s 2024-09-21T12:31:17.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 150743 10.45 MB/sec execute 235 sec latency 67.581 ms 2024-09-21T12:31:18.570 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:18 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:18.570 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:18 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:18.570 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:18 smithi078 ceph-mon[63866]: pgmap v42: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.5 MiB/s wr, 604 op/s 2024-09-21T12:31:18.570 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:18 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:18.570 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:18 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:18.678 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:18 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:18.678 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:18 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:18.678 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:18 smithi148 ceph-mon[49387]: pgmap v42: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.5 MiB/s wr, 604 op/s 2024-09-21T12:31:18.679 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:18 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:18.679 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:18 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:18.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 151294 10.46 MB/sec execute 236 sec latency 77.949 ms 2024-09-21T12:31:19.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 151766 10.47 MB/sec execute 237 sec latency 43.122 ms 2024-09-21T12:31:20.607 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:20 smithi078 ceph-mon[63866]: pgmap v43: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.1 MiB/s wr, 402 op/s 2024-09-21T12:31:20.659 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-21T12:31:20.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:20 smithi148 ceph-mon[49387]: pgmap v43: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.1 MiB/s wr, 402 op/s 2024-09-21T12:31:20.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 152103 10.49 MB/sec execute 238 sec latency 52.389 ms 2024-09-21T12:31:21.621 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:21 smithi078 ceph-mon[63866]: from='client.44121 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:31:21.747 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T12:31:21.748 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (3m) 72s ago 12m 20.7M - 0.27.0 11f11916f8cd e6e67ab07dec 2024-09-21T12:31:21.748 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 starting - - - - 2024-09-21T12:31:21.748 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi148 smithi148 running (8s) 4s ago 11m 6895k - 19.3.0-5104-g44178494 b415816db8b4 23deb0615621 2024-09-21T12:31:21.748 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (3m) 72s ago 12m 53.3M - dad864ee21e9 2a140c36091b 2024-09-21T12:31:21.748 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.ytpzwh smithi078 running (6m) 72s ago 6m 37.4M - 17.2.0 e1d6a67b021e 5e9b109502a2 2024-09-21T12:31:21.748 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi148.uukezr smithi148 running (6m) 4s ago 6m 13.7M - 17.2.0 e1d6a67b021e b7e8aa0f3a6c 2024-09-21T12:31:21.748 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.ntwyan smithi078 *:8443,9283,8765 running (2m) 72s ago 13m 488M - 19.3.0-5104-g44178494 b415816db8b4 a61baedebf43 2024-09-21T12:31:21.748 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi148.oyiqmx smithi148 *:8443,9283,8765 running (107s) 4s ago 11m 454M - 19.3.0-5104-g44178494 b415816db8b4 0255535c6fdb 2024-09-21T12:31:21.748 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (90s) 72s ago 13m 34.4M 2048M 19.3.0-5104-g44178494 b415816db8b4 22b3097efbd8 2024-09-21T12:31:21.748 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi148 smithi148 running (65s) 4s ago 11m 36.1M 2048M 19.3.0-5104-g44178494 b415816db8b4 df8b08509136 2024-09-21T12:31:21.748 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.ydbhru smithi078 *:2049 running (6m) 72s ago 6m 262M - 4.0 e1d6a67b021e 7e5b0274ac38 2024-09-21T12:31:21.749 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi148.einuzt smithi148 *:2049 running (6m) 4s ago 6m 86.8M - 4.0 e1d6a67b021e 76a8b0eabede 2024-09-21T12:31:21.749 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (3m) 72s ago 12m 11.5M - 1.7.0 72c9c2088986 301adb4e9e22 2024-09-21T12:31:21.749 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi148 smithi148 *:9100 running (2m) 4s ago 11m 11.3M - 1.7.0 72c9c2088986 0cceff4bf342 2024-09-21T12:31:21.749 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (11m) 72s ago 11m 453M 1936M 17.2.0 e1d6a67b021e d4d5e9977070 2024-09-21T12:31:21.749 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (10m) 72s ago 10m 402M 1936M 17.2.0 e1d6a67b021e b9a4b952233b 2024-09-21T12:31:21.749 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (10m) 72s ago 10m 308M 1936M 17.2.0 e1d6a67b021e 829fb8c2d827 2024-09-21T12:31:21.749 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (9m) 72s ago 9m 416M 1936M 17.2.0 e1d6a67b021e ef942c8556d0 2024-09-21T12:31:21.749 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi148 running (9m) 4s ago 9m 493M 2704M 17.2.0 e1d6a67b021e 62eefdd572c0 2024-09-21T12:31:21.749 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi148 running (8m) 4s ago 8m 314M 2704M 17.2.0 e1d6a67b021e 7b44be78baca 2024-09-21T12:31:21.749 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi148 running (8m) 4s ago 8m 509M 2704M 17.2.0 e1d6a67b021e 3d23057910ea 2024-09-21T12:31:21.749 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi148 running (8m) 4s ago 8m 334M 2704M 17.2.0 e1d6a67b021e f9e4db9d35fa 2024-09-21T12:31:21.749 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (114s) 72s ago 11m 47.4M - 2.51.0 1d3b7f56885b face2d4cdd1b 2024-09-21T12:31:21.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:21 smithi148 ceph-mon[49387]: from='client.44121 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:31:21.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 152457 10.49 MB/sec execute 239 sec latency 75.832 ms 2024-09-21T12:31:22.285 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:31:22.285 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-21T12:31:22.285 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:31:22.285 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:31:22.285 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-21T12:31:22.285 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:31:22.286 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:31:22.286 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-21T12:31:22.286 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-21T12:31:22.286 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:31:22.286 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-21T12:31:22.286 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:31:22.286 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:31:22.286 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-21T12:31:22.286 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-21T12:31:22.286 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 4 2024-09-21T12:31:22.286 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T12:31:22.286 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:31:22.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:22 smithi078 ceph-mon[63866]: from='client.34176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:31:22.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:22 smithi078 ceph-mon[63866]: pgmap v44: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.7 MiB/s wr, 680 op/s 2024-09-21T12:31:22.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:22 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:22.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:22 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:22.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:22 smithi078 ceph-mon[63866]: from='client.44129 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:31:22.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:22 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/3566286830' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:31:22.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:22 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:22.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:22 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:22.771 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:31:22.771 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5", 2024-09-21T12:31:22.771 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-21T12:31:22.772 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T12:31:22.772 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-21T12:31:22.772 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-21T12:31:22.772 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-09-21T12:31:22.772 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr" 2024-09-21T12:31:22.772 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-21T12:31:22.772 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "6/23 daemons upgraded", 2024-09-21T12:31:22.772 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading crash daemons", 2024-09-21T12:31:22.772 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-21T12:31:22.772 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:31:22.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:22 smithi148 ceph-mon[49387]: from='client.34176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:31:22.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:22 smithi148 ceph-mon[49387]: pgmap v44: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.7 MiB/s wr, 680 op/s 2024-09-21T12:31:22.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:22 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:22.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:22 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:22.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:22 smithi148 ceph-mon[49387]: from='client.44129 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:31:22.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:22 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/3566286830' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:31:22.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:22 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:22.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:22 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:22.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 152775 10.49 MB/sec execute 240 sec latency 70.853 ms 2024-09-21T12:31:23.369 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-21T12:31:23.669 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:23 smithi078 ceph-mon[63866]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:31:23.669 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:23 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:23.669 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:23 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:23.669 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:23 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/2199837588' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:31:23.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:23 smithi148 ceph-mon[49387]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:31:23.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:23 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:23.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:23 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:23.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:23 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/2199837588' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:31:23.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 153092 10.49 MB/sec execute 241 sec latency 55.303 ms 2024-09-21T12:31:24.684 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:24 smithi078 ceph-mon[63866]: pgmap v45: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.8 MiB/s wr, 464 op/s 2024-09-21T12:31:24.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:24 smithi148 ceph-mon[49387]: pgmap v45: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.8 MiB/s wr, 464 op/s 2024-09-21T12:31:24.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 153328 10.46 MB/sec execute 242 sec latency 89.236 ms 2024-09-21T12:31:25.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 153732 10.44 MB/sec execute 243 sec latency 47.658 ms 2024-09-21T12:31:26.726 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:26 smithi078 ceph-mon[63866]: pgmap v46: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.8 MiB/s wr, 464 op/s 2024-09-21T12:31:26.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:26 smithi148 ceph-mon[49387]: pgmap v46: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.8 MiB/s wr, 464 op/s 2024-09-21T12:31:26.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 154033 10.43 MB/sec execute 244 sec latency 57.212 ms 2024-09-21T12:31:27.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 154505 10.44 MB/sec execute 245 sec latency 38.538 ms 2024-09-21T12:31:28.765 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:28 smithi078 ceph-mon[63866]: pgmap v47: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.0 MiB/s wr, 659 op/s 2024-09-21T12:31:28.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:28 smithi148 ceph-mon[49387]: pgmap v47: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.0 MiB/s wr, 659 op/s 2024-09-21T12:31:28.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 154951 10.43 MB/sec execute 246 sec latency 64.221 ms 2024-09-21T12:31:29.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 155271 10.43 MB/sec execute 247 sec latency 59.037 ms 2024-09-21T12:31:30.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:30.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:30.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:31:30.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:31:30.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:30.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:30.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:30 smithi078 ceph-mon[63866]: pgmap v48: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 473 op/s 2024-09-21T12:31:30.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:31:30.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:31:30.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:31:30.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:31:30.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:30 smithi078 ceph-mon[63866]: Upgrade: Setting container_image for all crash 2024-09-21T12:31:30.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:30.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi078"} : dispatch 2024-09-21T12:31:30.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi078"}]': finished 2024-09-21T12:31:30.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi148"} : dispatch 2024-09-21T12:31:30.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi148"}]': finished 2024-09-21T12:31:30.308 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-21T12:31:30.309 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:30 smithi078 ceph-mon[63866]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-21T12:31:30.309 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:30 smithi078 ceph-mon[63866]: Upgrade: osd.0 is safe to restart 2024-09-21T12:31:30.309 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:31:30.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:30.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:30.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:31:30.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:31:30.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:30.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:30.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:30 smithi148 ceph-mon[49387]: pgmap v48: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 473 op/s 2024-09-21T12:31:30.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:31:30.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:31:30.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:31:30.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:31:30.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:30 smithi148 ceph-mon[49387]: Upgrade: Setting container_image for all crash 2024-09-21T12:31:30.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:30.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi078"} : dispatch 2024-09-21T12:31:30.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi078"}]': finished 2024-09-21T12:31:30.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi148"} : dispatch 2024-09-21T12:31:30.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi148"}]': finished 2024-09-21T12:31:30.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-21T12:31:30.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:30 smithi148 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-21T12:31:30.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:30 smithi148 ceph-mon[49387]: Upgrade: osd.0 is safe to restart 2024-09-21T12:31:30.552 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:31:30.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 155872 10.48 MB/sec execute 248 sec latency 27.649 ms 2024-09-21T12:31:31.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 156522 10.53 MB/sec execute 249 sec latency 19.153 ms 2024-09-21T12:31:32.691 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:32 smithi078 ceph-mon[63866]: pgmap v49: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 720 op/s 2024-09-21T12:31:32.691 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:32 smithi078 ceph-mon[63866]: mgrmap e43: smithi078.ntwyan(active, since 92s), standbys: smithi148.oyiqmx 2024-09-21T12:31:32.691 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:32 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:32.691 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:32 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T12:31:32.691 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:32 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:31:32.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:32 smithi148 ceph-mon[49387]: pgmap v49: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 720 op/s 2024-09-21T12:31:32.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:32 smithi148 ceph-mon[49387]: mgrmap e43: smithi078.ntwyan(active, since 92s), standbys: smithi148.oyiqmx 2024-09-21T12:31:32.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:32 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:32.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:32 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T12:31:32.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:32 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:31:32.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 156908 10.51 MB/sec execute 250 sec latency 46.373 ms 2024-09-21T12:31:33.715 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:33 smithi078 ceph-mon[63866]: Upgrade: Updating osd.0 2024-09-21T12:31:33.716 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:33 smithi078 ceph-mon[63866]: Deploying daemon osd.0 on smithi078 2024-09-21T12:31:33.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:33 smithi148 ceph-mon[49387]: Upgrade: Updating osd.0 2024-09-21T12:31:33.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:33 smithi148 ceph-mon[49387]: Deploying daemon osd.0 on smithi078 2024-09-21T12:31:33.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 157213 10.48 MB/sec execute 251 sec latency 151.308 ms 2024-09-21T12:31:34.736 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:34 smithi078 ceph-mon[63866]: pgmap v50: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.6 MiB/s wr, 441 op/s 2024-09-21T12:31:34.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:34 smithi148 ceph-mon[49387]: pgmap v50: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.6 MiB/s wr, 441 op/s 2024-09-21T12:31:34.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 157514 10.47 MB/sec execute 252 sec latency 49.944 ms 2024-09-21T12:31:35.804 INFO:teuthology.orchestra.run.smithi078.stdout: 5 157862 10.47 MB/sec execute 253 sec latency 103.128 ms 2024-09-21T12:31:36.757 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:36 smithi078 ceph-mon[63866]: pgmap v51: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.6 MiB/s wr, 441 op/s 2024-09-21T12:31:36.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:36 smithi148 ceph-mon[49387]: pgmap v51: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.6 MiB/s wr, 441 op/s 2024-09-21T12:31:36.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 158264 10.46 MB/sec execute 254 sec latency 60.646 ms 2024-09-21T12:31:37.522 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:37 smithi078 systemd[1]: Stopping Ceph osd.0 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:31:37.788 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:37 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0[34202]: 2024-09-21T12:31:37.723+0000 7ff993e67700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T12:31:37.788 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:37 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0[34202]: 2024-09-21T12:31:37.723+0000 7ff993e67700 -1 osd.0 72 *** Got signal Terminated *** 2024-09-21T12:31:37.788 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:37 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0[34202]: 2024-09-21T12:31:37.723+0000 7ff993e67700 -1 osd.0 72 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T12:31:37.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 158604 10.46 MB/sec execute 255 sec latency 70.845 ms 2024-09-21T12:31:38.547 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:38 smithi078 ceph-mon[63866]: pgmap v52: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 703 op/s 2024-09-21T12:31:38.547 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:38 smithi078 ceph-mon[63866]: osd.0 marked itself down and dead 2024-09-21T12:31:38.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:38 smithi148 ceph-mon[49387]: pgmap v52: 97 pgs: 97 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 703 op/s 2024-09-21T12:31:38.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:38 smithi148 ceph-mon[49387]: osd.0 marked itself down and dead 2024-09-21T12:31:38.804 INFO:teuthology.orchestra.run.smithi078.stdout: 5 158848 10.45 MB/sec execute 256 sec latency 268.288 ms 2024-09-21T12:31:39.481 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:39 smithi078 podman[68186]: 2024-09-21 12:31:39.188737902 +0000 UTC m=+1.588648060 container died d4d5e99770704aa35881feb08ddd875b80c2bdc555c149dea51e15c6f8d85bf0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, name=centos-stream, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, version=8, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, release=754, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, architecture=x86_64) 2024-09-21T12:31:39.732 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:39 smithi078 ceph-mon[63866]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T12:31:39.732 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:39 smithi078 ceph-mon[63866]: osdmap e73: 8 total, 7 up, 8 in 2024-09-21T12:31:39.732 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:39 smithi078 podman[68186]: 2024-09-21 12:31:39.60433698 +0000 UTC m=+2.004247124 container cleanup d4d5e99770704aa35881feb08ddd875b80c2bdc555c149dea51e15c6f8d85bf0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , name=centos-stream, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, version=8, distribution-scope=public, vcs-type=git, release=754, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-21T12:31:39.732 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:39 smithi078 bash[68186]: ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0 2024-09-21T12:31:39.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:39 smithi148 ceph-mon[49387]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T12:31:39.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:39 smithi148 ceph-mon[49387]: osdmap e73: 8 total, 7 up, 8 in 2024-09-21T12:31:39.803 INFO:teuthology.orchestra.run.smithi078.stdout: 5 158911 10.41 MB/sec execute 257 sec latency 1051.118 ms 2024-09-21T12:31:39.986 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:39 smithi078 podman[68201]: 2024-09-21 12:31:39.908526744 +0000 UTC m=+0.723537980 container remove d4d5e99770704aa35881feb08ddd875b80c2bdc555c149dea51e15c6f8d85bf0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, maintainer=Guillaume Abrioux , RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, release=754, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-21T12:31:40.494 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:40 smithi078 podman[68285]: 2024-09-21 12:31:40.310224068 +0000 UTC m=+0.019169618 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:31:40.494 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:40 smithi078 podman[68285]: 2024-09-21 12:31:40.433157063 +0000 UTC m=+0.142102611 container create d3bbfaaec2b230b9203a37c9f6af7a33829ecc2d4e1cd11b83dc954b0528c8d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0-deactivate, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0842-4417849, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-21T12:31:40.747 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:40 smithi078 ceph-mon[63866]: pgmap v54: 97 pgs: 13 stale+active+clean, 84 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.0 MiB/s wr, 610 op/s 2024-09-21T12:31:40.748 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:40 smithi078 ceph-mon[63866]: osdmap e74: 8 total, 7 up, 8 in 2024-09-21T12:31:40.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:40 smithi148 ceph-mon[49387]: pgmap v54: 97 pgs: 13 stale+active+clean, 84 active+clean; 337 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.0 MiB/s wr, 610 op/s 2024-09-21T12:31:40.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:40 smithi148 ceph-mon[49387]: osdmap e74: 8 total, 7 up, 8 in 2024-09-21T12:31:40.804 INFO:teuthology.orchestra.run.smithi078.stdout: 5 159201 10.41 MB/sec execute 258 sec latency 76.119 ms 2024-09-21T12:31:41.506 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:41 smithi078 podman[68285]: 2024-09-21 12:31:41.264307237 +0000 UTC m=+0.973252795 container init d3bbfaaec2b230b9203a37c9f6af7a33829ecc2d4e1cd11b83dc954b0528c8d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.license=GPLv2) 2024-09-21T12:31:41.506 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:41 smithi078 podman[68285]: 2024-09-21 12:31:41.276359966 +0000 UTC m=+0.985305506 container start d3bbfaaec2b230b9203a37c9f6af7a33829ecc2d4e1cd11b83dc954b0528c8d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2) 2024-09-21T12:31:41.506 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:41 smithi078 podman[68285]: 2024-09-21 12:31:41.476684209 +0000 UTC m=+1.185629748 container attach d3bbfaaec2b230b9203a37c9f6af7a33829ecc2d4e1cd11b83dc954b0528c8d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD) 2024-09-21T12:31:41.759 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:41 smithi078 ceph-mon[63866]: Health check failed: Degraded data redundancy: 206/1581 objects degraded (13.030%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T12:31:41.759 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:41 smithi078 podman[68285]: 2024-09-21 12:31:41.579870263 +0000 UTC m=+1.288815810 container died d3bbfaaec2b230b9203a37c9f6af7a33829ecc2d4e1cd11b83dc954b0528c8d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-21-0842-4417849) 2024-09-21T12:31:41.804 INFO:teuthology.orchestra.run.smithi078.stdout: 5 159469 10.41 MB/sec execute 259 sec latency 128.230 ms 2024-09-21T12:31:42.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:41 smithi148 ceph-mon[49387]: Health check failed: Degraded data redundancy: 206/1581 objects degraded (13.030%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T12:31:42.550 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:42 smithi078 podman[68308]: 2024-09-21 12:31:42.420479893 +0000 UTC m=+0.834637134 container remove d3bbfaaec2b230b9203a37c9f6af7a33829ecc2d4e1cd11b83dc954b0528c8d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0842-4417849, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-21T12:31:42.550 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:42 smithi078 systemd[1]: ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.0.service: Deactivated successfully. 2024-09-21T12:31:42.550 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:42 smithi078 systemd[1]: Stopped Ceph osd.0 for 6263deca-7813-11ef-baf6-efdc52797490. 2024-09-21T12:31:42.550 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:42 smithi078 systemd[1]: ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.0.service: Consumed 2min 35.683s CPU time. 2024-09-21T12:31:42.809 INFO:teuthology.orchestra.run.smithi078.stdout: 5 159794 10.42 MB/sec execute 260 sec latency 45.520 ms 2024-09-21T12:31:42.810 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:42 smithi078 systemd[1]: Starting Ceph osd.0 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:31:42.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:42 smithi078 ceph-mon[63866]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.3 MiB/s wr, 632 op/s; 206/1581 objects degraded (13.030%) 2024-09-21T12:31:43.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:42 smithi148 ceph-mon[49387]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.3 MiB/s wr, 632 op/s; 206/1581 objects degraded (13.030%) 2024-09-21T12:31:43.068 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:42 smithi078 podman[68400]: 2024-09-21 12:31:42.86744366 +0000 UTC m=+0.018442851 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:31:43.068 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:42 smithi078 podman[68400]: 2024-09-21 12:31:42.980069965 +0000 UTC m=+0.131069148 container create 1b1e90cbaf37fec9503a9c446e9b30ede81b4241a0afdfe15f320a4d8fe59eb2 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0-activate, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True) 2024-09-21T12:31:43.323 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:43 smithi078 podman[68400]: 2024-09-21 12:31:43.308150723 +0000 UTC m=+0.459149915 container init 1b1e90cbaf37fec9503a9c446e9b30ede81b4241a0afdfe15f320a4d8fe59eb2 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0-activate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0842-4417849, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T12:31:43.324 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:43 smithi078 podman[68400]: 2024-09-21 12:31:43.315780024 +0000 UTC m=+0.466779210 container start 1b1e90cbaf37fec9503a9c446e9b30ede81b4241a0afdfe15f320a4d8fe59eb2 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0-activate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-21-0842-4417849) 2024-09-21T12:31:43.578 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:43 smithi078 podman[68400]: 2024-09-21 12:31:43.458978169 +0000 UTC m=+0.609977354 container attach 1b1e90cbaf37fec9503a9c446e9b30ede81b4241a0afdfe15f320a4d8fe59eb2 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-21T12:31:43.578 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:43 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0-activate[68415]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:31:43.578 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:43 smithi078 bash[68400]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:31:43.578 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:43 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0-activate[68415]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:31:43.578 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:43 smithi078 bash[68400]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:31:43.809 INFO:teuthology.orchestra.run.smithi078.stdout: 5 160038 10.40 MB/sec execute 261 sec latency 49.548 ms 2024-09-21T12:31:44.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:44 smithi078 ceph-mon[63866]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.3 MiB/s wr, 632 op/s; 206/1581 objects degraded (13.030%) 2024-09-21T12:31:44.809 INFO:teuthology.orchestra.run.smithi078.stdout: 5 160310 10.39 MB/sec execute 262 sec latency 72.273 ms 2024-09-21T12:31:45.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:44 smithi148 ceph-mon[49387]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.3 MiB/s wr, 632 op/s; 206/1581 objects degraded (13.030%) 2024-09-21T12:31:45.809 INFO:teuthology.orchestra.run.smithi078.stdout: 5 160617 10.36 MB/sec execute 263 sec latency 57.586 ms 2024-09-21T12:31:45.903 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:45.903 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:31:45.903 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:45 smithi078 ceph-mon[63866]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 2.5 MiB/s wr, 239 op/s; 206/1581 objects degraded (13.030%) 2024-09-21T12:31:46.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:46.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:31:46.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:45 smithi148 ceph-mon[49387]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 2.5 MiB/s wr, 239 op/s; 206/1581 objects degraded (13.030%) 2024-09-21T12:31:46.809 INFO:teuthology.orchestra.run.smithi078.stdout: 5 160910 10.35 MB/sec execute 264 sec latency 83.346 ms 2024-09-21T12:31:47.711 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:47 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 220/1695 objects degraded (12.979%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T12:31:47.809 INFO:teuthology.orchestra.run.smithi078.stdout: 5 161261 10.34 MB/sec execute 265 sec latency 55.778 ms 2024-09-21T12:31:47.968 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:47 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0-activate[68415]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T12:31:47.968 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:47 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0-activate[68415]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:31:47.968 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:47 smithi078 bash[68400]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T12:31:47.968 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:47 smithi078 bash[68400]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:31:47.968 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:47 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0-activate[68415]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:31:47.968 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:47 smithi078 bash[68400]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:31:48.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:47 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 220/1695 objects degraded (12.979%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T12:31:48.228 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:48 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0-activate[68415]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T12:31:48.228 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:48 smithi078 bash[68400]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T12:31:48.228 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:48 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0-activate[68415]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-09-21T12:31:48.228 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:48 smithi078 bash[68400]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-09-21T12:31:48.502 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:48 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0-activate[68415]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-21T12:31:48.502 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:48 smithi078 bash[68400]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-21T12:31:48.502 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:48 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0-activate[68415]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-09-21T12:31:48.502 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:48 smithi078 bash[68400]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-09-21T12:31:48.502 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:48 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0-activate[68415]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T12:31:48.502 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:48 smithi078 bash[68400]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T12:31:48.502 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:48 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0-activate[68415]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T12:31:48.502 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:48 smithi078 bash[68400]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T12:31:48.503 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:48 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0-activate[68415]: --> ceph-volume lvm activate successful for osd ID: 0 2024-09-21T12:31:48.503 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:48 smithi078 bash[68400]: --> ceph-volume lvm activate successful for osd ID: 0 2024-09-21T12:31:48.743 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:48 smithi078 ceph-mon[63866]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.7 MiB/s wr, 438 op/s; 220/1695 objects degraded (12.979%) 2024-09-21T12:31:48.744 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:48 smithi078 podman[68400]: 2024-09-21 12:31:48.509861586 +0000 UTC m=+5.660860777 container died 1b1e90cbaf37fec9503a9c446e9b30ede81b4241a0afdfe15f320a4d8fe59eb2 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0-activate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-21-0842-4417849, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-21T12:31:48.809 INFO:teuthology.orchestra.run.smithi078.stdout: 5 161665 10.34 MB/sec execute 266 sec latency 83.380 ms 2024-09-21T12:31:49.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:48 smithi148 ceph-mon[49387]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.7 MiB/s wr, 438 op/s; 220/1695 objects degraded (12.979%) 2024-09-21T12:31:49.520 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:49 smithi078 podman[68637]: 2024-09-21 12:31:49.392334623 +0000 UTC m=+0.875283136 container remove 1b1e90cbaf37fec9503a9c446e9b30ede81b4241a0afdfe15f320a4d8fe59eb2 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0-activate, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-21T12:31:49.771 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:49 smithi078 podman[68691]: 2024-09-21 12:31:49.533945389 +0000 UTC m=+0.021043547 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:31:49.771 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:49 smithi078 podman[68691]: 2024-09-21 12:31:49.684900049 +0000 UTC m=+0.171998194 container create 7374add9753462c8ad5b0ccb253024e1501e9e80b11efd84c83f38e19b61d493 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-21T12:31:49.809 INFO:teuthology.orchestra.run.smithi078.stdout: 5 162067 10.34 MB/sec execute 267 sec latency 48.057 ms 2024-09-21T12:31:50.296 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:50 smithi078 podman[68691]: 2024-09-21 12:31:50.022527663 +0000 UTC m=+0.509625819 container init 7374add9753462c8ad5b0ccb253024e1501e9e80b11efd84c83f38e19b61d493 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-21-0842-4417849, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-21T12:31:50.296 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:50 smithi078 podman[68691]: 2024-09-21 12:31:50.033426014 +0000 UTC m=+0.520524159 container start 7374add9753462c8ad5b0ccb253024e1501e9e80b11efd84c83f38e19b61d493 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-21T12:31:50.296 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:50 smithi078 ceph-osd[68709]: -- 172.21.15.78:0/2499375568 <== mon.0 v2:172.21.15.78:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x558cbbe481e0 con 0x558cbafe3c00 2024-09-21T12:31:50.296 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:50 smithi078 bash[68691]: 7374add9753462c8ad5b0ccb253024e1501e9e80b11efd84c83f38e19b61d493 2024-09-21T12:31:50.296 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:50 smithi078 systemd[1]: Started Ceph osd.0 for 6263deca-7813-11ef-baf6-efdc52797490. 2024-09-21T12:31:50.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:50 smithi078 ceph-mon[63866]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.2 MiB/s wr, 391 op/s; 220/1695 objects degraded (12.979%) 2024-09-21T12:31:50.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:50 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:50.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:50 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:50.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:50 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:31:50.809 INFO:teuthology.orchestra.run.smithi078.stdout: 5 162373 10.33 MB/sec execute 268 sec latency 81.234 ms 2024-09-21T12:31:51.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:50 smithi148 ceph-mon[49387]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 341 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.2 MiB/s wr, 391 op/s; 220/1695 objects degraded (12.979%) 2024-09-21T12:31:51.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:50 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:51.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:50 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:51.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:50 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:31:51.819 INFO:teuthology.orchestra.run.smithi078.stdout: 5 162776 10.34 MB/sec execute 269 sec latency 45.029 ms 2024-09-21T12:31:52.078 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:51 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0[68705]: 2024-09-21T12:31:51.830+0000 7f9f3fc17640 -1 Falling back to public interface 2024-09-21T12:31:52.819 INFO:teuthology.orchestra.run.smithi078.stdout: 5 163108 10.35 MB/sec execute 270 sec latency 104.568 ms 2024-09-21T12:31:52.846 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:52 smithi078 ceph-mon[63866]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.5 MiB/s wr, 549 op/s; 224/1635 objects degraded (13.700%) 2024-09-21T12:31:53.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:52 smithi148 ceph-mon[49387]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.5 MiB/s wr, 549 op/s; 224/1635 objects degraded (13.700%) 2024-09-21T12:31:53.819 INFO:teuthology.orchestra.run.smithi078.stdout: 5 163418 10.34 MB/sec execute 271 sec latency 54.705 ms 2024-09-21T12:31:53.871 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-21T12:31:54.635 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:54 smithi078 ceph-mon[63866]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 386 op/s; 224/1635 objects degraded (13.700%) 2024-09-21T12:31:54.763 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T12:31:54.763 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (4m) 32s ago 13m 20.7M - 0.27.0 11f11916f8cd e6e67ab07dec 2024-09-21T12:31:54.763 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (47s) 32s ago 13m 6895k - 19.3.0-5104-g44178494 b415816db8b4 46a10ea0b111 2024-09-21T12:31:54.764 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi148 smithi148 running (41s) 37s ago 12m 6895k - 19.3.0-5104-g44178494 b415816db8b4 23deb0615621 2024-09-21T12:31:54.764 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (3m) 32s ago 12m 52.5M - dad864ee21e9 2a140c36091b 2024-09-21T12:31:54.764 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.ytpzwh smithi078 running (7m) 32s ago 7m 37.7M - 17.2.0 e1d6a67b021e 5e9b109502a2 2024-09-21T12:31:54.764 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi148.uukezr smithi148 running (7m) 37s ago 7m 13.7M - 17.2.0 e1d6a67b021e b7e8aa0f3a6c 2024-09-21T12:31:54.764 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.ntwyan smithi078 *:8443,9283,8765 running (3m) 32s ago 14m 509M - 19.3.0-5104-g44178494 b415816db8b4 a61baedebf43 2024-09-21T12:31:54.764 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi148.oyiqmx smithi148 *:8443,9283,8765 running (2m) 37s ago 12m 454M - 19.3.0-5104-g44178494 b415816db8b4 0255535c6fdb 2024-09-21T12:31:54.764 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (2m) 32s ago 14m 47.6M 2048M 19.3.0-5104-g44178494 b415816db8b4 22b3097efbd8 2024-09-21T12:31:54.764 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi148 smithi148 running (98s) 37s ago 12m 36.1M 2048M 19.3.0-5104-g44178494 b415816db8b4 df8b08509136 2024-09-21T12:31:54.764 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.ydbhru smithi078 *:2049 running (6m) 32s ago 6m 264M - 4.0 e1d6a67b021e 7e5b0274ac38 2024-09-21T12:31:54.764 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi148.einuzt smithi148 *:2049 running (6m) 37s ago 6m 86.8M - 4.0 e1d6a67b021e 76a8b0eabede 2024-09-21T12:31:54.764 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (3m) 32s ago 12m 10.9M - 1.7.0 72c9c2088986 301adb4e9e22 2024-09-21T12:31:54.764 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi148 smithi148 *:9100 running (3m) 37s ago 12m 11.3M - 1.7.0 72c9c2088986 0cceff4bf342 2024-09-21T12:31:54.764 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 starting - - - 1936M 2024-09-21T12:31:54.764 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (11m) 32s ago 11m 461M 1936M 17.2.0 e1d6a67b021e b9a4b952233b 2024-09-21T12:31:54.765 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (10m) 32s ago 10m 380M 1936M 17.2.0 e1d6a67b021e 829fb8c2d827 2024-09-21T12:31:54.765 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (10m) 32s ago 10m 486M 1936M 17.2.0 e1d6a67b021e ef942c8556d0 2024-09-21T12:31:54.765 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi148 running (9m) 37s ago 9m 493M 2704M 17.2.0 e1d6a67b021e 62eefdd572c0 2024-09-21T12:31:54.765 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi148 running (9m) 37s ago 9m 314M 2704M 17.2.0 e1d6a67b021e 7b44be78baca 2024-09-21T12:31:54.765 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi148 running (9m) 37s ago 9m 509M 2704M 17.2.0 e1d6a67b021e 3d23057910ea 2024-09-21T12:31:54.765 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi148 running (8m) 37s ago 8m 334M 2704M 17.2.0 e1d6a67b021e f9e4db9d35fa 2024-09-21T12:31:54.765 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (2m) 32s ago 12m 49.2M - 2.51.0 1d3b7f56885b face2d4cdd1b 2024-09-21T12:31:54.819 INFO:teuthology.orchestra.run.smithi078.stdout: 5 163705 10.35 MB/sec execute 272 sec latency 76.929 ms 2024-09-21T12:31:55.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:54 smithi148 ceph-mon[49387]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 386 op/s; 224/1635 objects degraded (13.700%) 2024-09-21T12:31:55.336 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:31:55.336 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-21T12:31:55.337 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:31:55.337 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:31:55.337 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-21T12:31:55.337 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:31:55.337 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:31:55.337 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-21T12:31:55.337 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7 2024-09-21T12:31:55.337 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:31:55.337 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-21T12:31:55.337 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:31:55.337 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:31:55.337 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-21T12:31:55.338 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-21T12:31:55.338 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 4 2024-09-21T12:31:55.338 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T12:31:55.338 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:31:55.661 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:55 smithi078 ceph-mon[63866]: from='client.34198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:31:55.662 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:55 smithi078 ceph-mon[63866]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:31:55.662 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:55 smithi078 ceph-mon[63866]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:31:55.662 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:55 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/2173613419' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:31:55.819 INFO:teuthology.orchestra.run.smithi078.stdout: 5 163914 10.32 MB/sec execute 273 sec latency 56.696 ms 2024-09-21T12:31:55.938 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:31:55.938 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5", 2024-09-21T12:31:55.938 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-21T12:31:55.938 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T12:31:55.938 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-21T12:31:55.939 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-21T12:31:55.939 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-09-21T12:31:55.939 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr" 2024-09-21T12:31:55.939 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-21T12:31:55.939 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "6/23 daemons upgraded", 2024-09-21T12:31:55.939 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T12:31:55.939 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-21T12:31:55.939 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:31:56.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:55 smithi148 ceph-mon[49387]: from='client.34198 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:31:56.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:55 smithi148 ceph-mon[49387]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:31:56.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:55 smithi148 ceph-mon[49387]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:31:56.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:55 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/2173613419' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:31:56.692 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:56 smithi078 ceph-mon[63866]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 386 op/s; 224/1635 objects degraded (13.700%) 2024-09-21T12:31:56.694 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 224/1635 objects degraded (13.700%), 29 pgs degraded 2024-09-21T12:31:56.694 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-21T12:31:56.695 INFO:teuthology.orchestra.run.smithi078.stdout: osd.0 (root=default,host=smithi078) is down 2024-09-21T12:31:56.695 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 224/1635 objects degraded (13.700%), 29 pgs degraded 2024-09-21T12:31:56.695 INFO:teuthology.orchestra.run.smithi078.stdout: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-09-21T12:31:56.695 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.0 is active+undersized+degraded, acting [7,1] 2024-09-21T12:31:56.695 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1 is active+undersized+degraded, acting [2,3] 2024-09-21T12:31:56.695 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.4 is active+undersized+degraded, acting [1,7] 2024-09-21T12:31:56.695 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.5 is active+undersized+degraded, acting [7,4] 2024-09-21T12:31:56.695 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.8 is active+undersized+degraded, acting [7,1] 2024-09-21T12:31:56.695 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.c is active+undersized+degraded, acting [2,5] 2024-09-21T12:31:56.695 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.f is active+undersized+degraded, acting [4,7] 2024-09-21T12:31:56.695 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.10 is active+undersized+degraded, acting [2,1] 2024-09-21T12:31:56.695 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.13 is active+undersized+degraded, acting [4,3] 2024-09-21T12:31:56.695 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.15 is active+undersized+degraded, acting [1,5] 2024-09-21T12:31:56.696 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.19 is active+undersized+degraded, acting [3,6] 2024-09-21T12:31:56.696 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1d is active+undersized+degraded, acting [7,6] 2024-09-21T12:31:56.696 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1e is active+undersized+degraded, acting [3,5] 2024-09-21T12:31:56.696 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1f is active+undersized+degraded, acting [7,4] 2024-09-21T12:31:56.696 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-09-21T12:31:56.696 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-09-21T12:31:56.696 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-09-21T12:31:56.696 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-09-21T12:31:56.696 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.b is active+undersized+degraded, acting [3,4] 2024-09-21T12:31:56.696 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.f is active+undersized+degraded, acting [7,4] 2024-09-21T12:31:56.696 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-09-21T12:31:56.696 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-09-21T12:31:56.696 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-09-21T12:31:56.696 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-09-21T12:31:56.697 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-09-21T12:31:56.697 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-09-21T12:31:56.697 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-09-21T12:31:56.697 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.19 is active+undersized+degraded, acting [3,2] 2024-09-21T12:31:56.819 INFO:teuthology.orchestra.run.smithi078.stdout: 5 164162 10.30 MB/sec execute 274 sec latency 51.881 ms 2024-09-21T12:31:57.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:56 smithi148 ceph-mon[49387]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 333 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 386 op/s; 224/1635 objects degraded (13.700%) 2024-09-21T12:31:57.716 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:57 smithi078 ceph-mon[63866]: from='client.34214 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:31:57.716 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:57 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/3765537798' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:31:57.716 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:57 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 224/1635 objects degraded (13.700%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T12:31:57.819 INFO:teuthology.orchestra.run.smithi078.stdout: 5 164370 10.28 MB/sec execute 275 sec latency 56.162 ms 2024-09-21T12:31:58.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:57 smithi148 ceph-mon[49387]: from='client.34214 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:31:58.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:57 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/3765537798' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:31:58.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:57 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 224/1635 objects degraded (13.700%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T12:31:58.753 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:58 smithi078 ceph-mon[63866]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.0 MiB/s wr, 573 op/s; 225/1683 objects degraded (13.369%) 2024-09-21T12:31:58.753 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:58 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:58.753 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:31:58 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:58.819 INFO:teuthology.orchestra.run.smithi078.stdout: 5 164651 10.26 MB/sec execute 276 sec latency 72.242 ms 2024-09-21T12:31:59.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:58 smithi148 ceph-mon[49387]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.0 MiB/s wr, 573 op/s; 225/1683 objects degraded (13.369%) 2024-09-21T12:31:59.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:58 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:59.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:31:58 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:31:59.771 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:31:59 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0[68705]: 2024-09-21T12:31:59.601+0000 7f9f3fc17640 -1 osd.0 0 read_superblock omap replica is missing. 2024-09-21T12:31:59.819 INFO:teuthology.orchestra.run.smithi078.stdout: 5 164920 10.25 MB/sec execute 277 sec latency 51.253 ms 2024-09-21T12:32:00.539 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:00.539 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:00.539 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:00 smithi078 ceph-mon[63866]: pgmap v65: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 406 op/s; 225/1683 objects degraded (13.369%) 2024-09-21T12:32:00.539 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:32:00.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:00.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:00.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:00 smithi148 ceph-mon[49387]: pgmap v65: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 406 op/s; 225/1683 objects degraded (13.369%) 2024-09-21T12:32:00.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:32:00.819 INFO:teuthology.orchestra.run.smithi078.stdout: 5 165153 10.23 MB/sec execute 278 sec latency 67.232 ms 2024-09-21T12:32:01.003 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:32:00 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0[68705]: 2024-09-21T12:32:00.836+0000 7f9f3fc17640 -1 osd.0 72 log_to_monitors true 2024-09-21T12:32:01.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:01 smithi078 ceph-mon[63866]: from='osd.0 [v2:172.21.15.78:6802/3938106064,v1:172.21.15.78:6803/3938106064]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T12:32:01.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:01 smithi148 ceph-mon[49387]: from='osd.0 [v2:172.21.15.78:6802/3938106064,v1:172.21.15.78:6803/3938106064]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T12:32:01.819 INFO:teuthology.orchestra.run.smithi078.stdout: 5 165443 10.23 MB/sec execute 279 sec latency 128.298 ms 2024-09-21T12:32:02.076 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:32:02 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0[68705]: 2024-09-21T12:32:02.012+0000 7f9f339b9640 -1 osd.0 72 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T12:32:02.579 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:02 smithi078 ceph-mon[63866]: from='osd.0 [v2:172.21.15.78:6802/3938106064,v1:172.21.15.78:6803/3938106064]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T12:32:02.579 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:02 smithi078 ceph-mon[63866]: osdmap e75: 8 total, 7 up, 8 in 2024-09-21T12:32:02.579 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:02 smithi078 ceph-mon[63866]: from='osd.0 [v2:172.21.15.78:6802/3938106064,v1:172.21.15.78:6803/3938106064]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]} : dispatch 2024-09-21T12:32:02.579 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:02 smithi078 ceph-mon[63866]: pgmap v67: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.4 MiB/s wr, 419 op/s; 195/1539 objects degraded (12.671%) 2024-09-21T12:32:02.579 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:02 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 225/1683 objects degraded (13.369%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T12:32:02.579 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:02 smithi078 ceph-mon[63866]: from='osd.0 [v2:172.21.15.78:6802/3938106064,v1:172.21.15.78:6803/3938106064]' entity='osd.0' 2024-09-21T12:32:02.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:02 smithi148 ceph-mon[49387]: from='osd.0 [v2:172.21.15.78:6802/3938106064,v1:172.21.15.78:6803/3938106064]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T12:32:02.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:02 smithi148 ceph-mon[49387]: osdmap e75: 8 total, 7 up, 8 in 2024-09-21T12:32:02.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:02 smithi148 ceph-mon[49387]: from='osd.0 [v2:172.21.15.78:6802/3938106064,v1:172.21.15.78:6803/3938106064]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi078", "root=default"]} : dispatch 2024-09-21T12:32:02.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:02 smithi148 ceph-mon[49387]: pgmap v67: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 331 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.4 MiB/s wr, 419 op/s; 195/1539 objects degraded (12.671%) 2024-09-21T12:32:02.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:02 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 225/1683 objects degraded (13.369%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T12:32:02.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:02 smithi148 ceph-mon[49387]: from='osd.0 [v2:172.21.15.78:6802/3938106064,v1:172.21.15.78:6803/3938106064]' entity='osd.0' 2024-09-21T12:32:02.820 INFO:teuthology.orchestra.run.smithi078.stdout: 5 165768 10.22 MB/sec execute 280 sec latency 101.861 ms 2024-09-21T12:32:03.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:03 smithi148 ceph-mon[49387]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T12:32:03.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:03 smithi148 ceph-mon[49387]: osd.0 [v2:172.21.15.78:6802/3938106064,v1:172.21.15.78:6803/3938106064] boot 2024-09-21T12:32:03.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:03 smithi148 ceph-mon[49387]: osdmap e76: 8 total, 8 up, 8 in 2024-09-21T12:32:03.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:03 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T12:32:03.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:03 smithi078 ceph-mon[63866]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T12:32:03.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:03 smithi078 ceph-mon[63866]: osd.0 [v2:172.21.15.78:6802/3938106064,v1:172.21.15.78:6803/3938106064] boot 2024-09-21T12:32:03.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:03 smithi078 ceph-mon[63866]: osdmap e76: 8 total, 8 up, 8 in 2024-09-21T12:32:03.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:03 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T12:32:03.819 INFO:teuthology.orchestra.run.smithi078.stdout: 5 165914 10.19 MB/sec execute 281 sec latency 815.806 ms 2024-09-21T12:32:04.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:04 smithi148 ceph-mon[49387]: pgmap v69: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.6 MiB/s wr, 524 op/s; 195/1539 objects degraded (12.671%) 2024-09-21T12:32:04.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:04 smithi148 ceph-mon[49387]: osdmap e77: 8 total, 8 up, 8 in 2024-09-21T12:32:04.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:04 smithi078 ceph-mon[63866]: pgmap v69: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.6 MiB/s wr, 524 op/s; 195/1539 objects degraded (12.671%) 2024-09-21T12:32:04.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:04 smithi078 ceph-mon[63866]: osdmap e77: 8 total, 8 up, 8 in 2024-09-21T12:32:04.819 INFO:teuthology.orchestra.run.smithi078.stdout: 5 165914 10.16 MB/sec execute 282 sec latency 1815.829 ms 2024-09-21T12:32:05.820 INFO:teuthology.orchestra.run.smithi078.stdout: 5 166085 10.13 MB/sec execute 283 sec latency 2065.076 ms 2024-09-21T12:32:06.036 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:05 smithi078 ceph-mon[63866]: osdmap e78: 8 total, 8 up, 8 in 2024-09-21T12:32:06.036 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:05 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:06.036 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:05 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:06.036 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:05 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:32:06.036 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:05 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:32:06.036 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:05 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:06.036 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:05 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:06.036 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:05 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:32:06.036 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:05 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:32:06.037 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:05 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:32:06.037 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:05 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:32:06.037 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:05 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T12:32:06.037 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:05 smithi078 ceph-mon[63866]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T12:32:06.037 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:05 smithi078 ceph-mon[63866]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-21T12:32:06.037 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:05 smithi078 ceph-mon[63866]: pgmap v72: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 195/1539 objects degraded (12.671%) 2024-09-21T12:32:06.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:05 smithi148 ceph-mon[49387]: osdmap e78: 8 total, 8 up, 8 in 2024-09-21T12:32:06.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:05 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:06.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:05 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:06.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:05 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:32:06.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:05 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:32:06.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:05 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:06.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:05 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:06.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:05 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:32:06.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:05 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:32:06.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:05 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:32:06.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:05 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:32:06.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:05 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T12:32:06.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:05 smithi148 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T12:32:06.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:05 smithi148 ceph-mon[49387]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-21T12:32:06.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:05 smithi148 ceph-mon[49387]: pgmap v72: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 195/1539 objects degraded (12.671%) 2024-09-21T12:32:06.820 INFO:teuthology.orchestra.run.smithi078.stdout: 5 166350 10.13 MB/sec execute 284 sec latency 70.214 ms 2024-09-21T12:32:07.082 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:07 smithi078 ceph-mon[63866]: osdmap e79: 8 total, 8 up, 8 in 2024-09-21T12:32:07.083 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:07 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 195/1539 objects degraded (12.671%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T12:32:07.083 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:07 smithi078 ceph-mon[63866]: osdmap e80: 8 total, 8 up, 8 in 2024-09-21T12:32:07.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:07 smithi148 ceph-mon[49387]: osdmap e79: 8 total, 8 up, 8 in 2024-09-21T12:32:07.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:07 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 195/1539 objects degraded (12.671%), 29 pgs degraded (PG_DEGRADED) 2024-09-21T12:32:07.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:07 smithi148 ceph-mon[49387]: osdmap e80: 8 total, 8 up, 8 in 2024-09-21T12:32:07.820 INFO:teuthology.orchestra.run.smithi078.stdout: 5 166516 10.11 MB/sec execute 285 sec latency 877.731 ms 2024-09-21T12:32:08.116 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:08 smithi078 ceph-mon[63866]: pgmap v75: 97 pgs: 9 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 85 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.9 MiB/s wr, 326 op/s; 203/1521 objects degraded (13.346%); 237 KiB/s, 14 objects/s recovering 2024-09-21T12:32:08.117 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:08 smithi078 ceph-mon[63866]: osdmap e81: 8 total, 8 up, 8 in 2024-09-21T12:32:08.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:08 smithi148 ceph-mon[49387]: pgmap v75: 97 pgs: 9 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 85 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.9 MiB/s wr, 326 op/s; 203/1521 objects degraded (13.346%); 237 KiB/s, 14 objects/s recovering 2024-09-21T12:32:08.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:08 smithi148 ceph-mon[49387]: osdmap e81: 8 total, 8 up, 8 in 2024-09-21T12:32:08.820 INFO:teuthology.orchestra.run.smithi078.stdout: 5 166787 10.12 MB/sec execute 286 sec latency 1002.339 ms 2024-09-21T12:32:09.820 INFO:teuthology.orchestra.run.smithi078.stdout: 5 166992 10.10 MB/sec execute 287 sec latency 705.130 ms 2024-09-21T12:32:10.164 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:09 smithi078 ceph-mon[63866]: osdmap e82: 8 total, 8 up, 8 in 2024-09-21T12:32:10.164 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:09 smithi078 ceph-mon[63866]: pgmap v78: 97 pgs: 9 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 85 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.9 MiB/s wr, 326 op/s; 203/1521 objects degraded (13.346%); 237 KiB/s, 14 objects/s recovering 2024-09-21T12:32:10.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:09 smithi148 ceph-mon[49387]: osdmap e82: 8 total, 8 up, 8 in 2024-09-21T12:32:10.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:09 smithi148 ceph-mon[49387]: pgmap v78: 97 pgs: 9 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 85 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.9 MiB/s wr, 326 op/s; 203/1521 objects degraded (13.346%); 237 KiB/s, 14 objects/s recovering 2024-09-21T12:32:10.820 INFO:teuthology.orchestra.run.smithi078.stdout: 5 167191 10.09 MB/sec execute 288 sec latency 847.381 ms 2024-09-21T12:32:11.188 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:10 smithi078 ceph-mon[63866]: osdmap e83: 8 total, 8 up, 8 in 2024-09-21T12:32:11.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:10 smithi148 ceph-mon[49387]: osdmap e83: 8 total, 8 up, 8 in 2024-09-21T12:32:11.820 INFO:teuthology.orchestra.run.smithi078.stdout: 5 167441 10.08 MB/sec execute 289 sec latency 651.731 ms 2024-09-21T12:32:12.215 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:11 smithi078 ceph-mon[63866]: osdmap e84: 8 total, 8 up, 8 in 2024-09-21T12:32:12.216 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:11 smithi078 ceph-mon[63866]: pgmap v81: 97 pgs: 2 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 89 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 4.9 MiB/s wr, 404 op/s; 89/1635 objects degraded (5.443%); 0 B/s, 25 objects/s recovering 2024-09-21T12:32:12.216 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:11 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 203/1521 objects degraded (13.346%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T12:32:12.216 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:11 smithi078 ceph-mon[63866]: osdmap e85: 8 total, 8 up, 8 in 2024-09-21T12:32:12.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:11 smithi148 ceph-mon[49387]: osdmap e84: 8 total, 8 up, 8 in 2024-09-21T12:32:12.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:11 smithi148 ceph-mon[49387]: pgmap v81: 97 pgs: 2 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 89 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 4.9 MiB/s wr, 404 op/s; 89/1635 objects degraded (5.443%); 0 B/s, 25 objects/s recovering 2024-09-21T12:32:12.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:11 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 203/1521 objects degraded (13.346%), 11 pgs degraded (PG_DEGRADED) 2024-09-21T12:32:12.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:11 smithi148 ceph-mon[49387]: osdmap e85: 8 total, 8 up, 8 in 2024-09-21T12:32:12.820 INFO:teuthology.orchestra.run.smithi078.stdout: 5 167594 10.05 MB/sec execute 290 sec latency 773.860 ms 2024-09-21T12:32:13.820 INFO:teuthology.orchestra.run.smithi078.stdout: 5 167798 10.04 MB/sec execute 291 sec latency 906.999 ms 2024-09-21T12:32:14.093 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:13 smithi078 ceph-mon[63866]: osdmap e86: 8 total, 8 up, 8 in 2024-09-21T12:32:14.093 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:13 smithi078 ceph-mon[63866]: pgmap v84: 97 pgs: 1 peering, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 5.0 MiB/s wr, 415 op/s; 43/1635 objects degraded (2.630%); 0 B/s, 40 objects/s recovering 2024-09-21T12:32:14.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:13 smithi148 ceph-mon[49387]: osdmap e86: 8 total, 8 up, 8 in 2024-09-21T12:32:14.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:13 smithi148 ceph-mon[49387]: pgmap v84: 97 pgs: 1 peering, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 5.0 MiB/s wr, 415 op/s; 43/1635 objects degraded (2.630%); 0 B/s, 40 objects/s recovering 2024-09-21T12:32:14.820 INFO:teuthology.orchestra.run.smithi078.stdout: 5 168038 10.01 MB/sec execute 292 sec latency 75.647 ms 2024-09-21T12:32:15.820 INFO:teuthology.orchestra.run.smithi078.stdout: 5 168306 10.00 MB/sec execute 293 sec latency 52.944 ms 2024-09-21T12:32:15.881 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:15 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:15.882 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:15 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:32:15.882 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:15 smithi078 ceph-mon[63866]: pgmap v85: 97 pgs: 1 peering, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.6 MiB/s wr, 299 op/s; 43/1635 objects degraded (2.630%); 0 B/s, 29 objects/s recovering 2024-09-21T12:32:16.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:15 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:16.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:15 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:32:16.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:15 smithi148 ceph-mon[49387]: pgmap v85: 97 pgs: 1 peering, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.6 MiB/s wr, 299 op/s; 43/1635 objects degraded (2.630%); 0 B/s, 29 objects/s recovering 2024-09-21T12:32:16.820 INFO:teuthology.orchestra.run.smithi078.stdout: 5 168511 9.98 MB/sec execute 294 sec latency 79.035 ms 2024-09-21T12:32:17.222 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:16 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 43/1635 objects degraded (2.630%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T12:32:17.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:16 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 43/1635 objects degraded (2.630%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T12:32:17.820 INFO:teuthology.orchestra.run.smithi078.stdout: 5 168735 9.97 MB/sec execute 295 sec latency 71.521 ms 2024-09-21T12:32:17.992 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:17 smithi078 ceph-mon[63866]: pgmap v86: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 2.2 MiB/s wr, 264 op/s; 0 B/s, 16 objects/s recovering 2024-09-21T12:32:18.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:17 smithi148 ceph-mon[49387]: pgmap v86: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 2.2 MiB/s wr, 264 op/s; 0 B/s, 16 objects/s recovering 2024-09-21T12:32:18.820 INFO:teuthology.orchestra.run.smithi078.stdout: 5 169119 9.99 MB/sec execute 296 sec latency 61.725 ms 2024-09-21T12:32:19.034 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:18 smithi078 ceph-mon[63866]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 43/1635 objects degraded (2.630%), 3 pgs degraded) 2024-09-21T12:32:19.034 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:18 smithi078 ceph-mon[63866]: Cluster is now healthy 2024-09-21T12:32:19.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:18 smithi148 ceph-mon[49387]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 43/1635 objects degraded (2.630%), 3 pgs degraded) 2024-09-21T12:32:19.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:18 smithi148 ceph-mon[49387]: Cluster is now healthy 2024-09-21T12:32:19.820 INFO:teuthology.orchestra.run.smithi078.stdout: 5 169590 9.98 MB/sec execute 297 sec latency 35.303 ms 2024-09-21T12:32:20.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:19 smithi078 ceph-mon[63866]: pgmap v87: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 1.8 MiB/s wr, 216 op/s; 0 B/s, 13 objects/s recovering 2024-09-21T12:32:20.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:19 smithi148 ceph-mon[49387]: pgmap v87: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 1.8 MiB/s wr, 216 op/s; 0 B/s, 13 objects/s recovering 2024-09-21T12:32:20.820 INFO:teuthology.orchestra.run.smithi078.stdout: 5 169920 9.98 MB/sec execute 298 sec latency 43.305 ms 2024-09-21T12:32:21.079 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:20 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T12:32:21.080 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:20 smithi078 ceph-mon[63866]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T12:32:21.080 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:20 smithi078 ceph-mon[63866]: Upgrade: osd.1 is safe to restart 2024-09-21T12:32:21.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:20 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T12:32:21.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:20 smithi148 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T12:32:21.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:20 smithi148 ceph-mon[49387]: Upgrade: osd.1 is safe to restart 2024-09-21T12:32:21.820 INFO:teuthology.orchestra.run.smithi078.stdout: 5 170300 10.00 MB/sec execute 299 sec latency 71.853 ms 2024-09-21T12:32:22.089 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:22 smithi078 ceph-mon[63866]: pgmap v88: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.6 MiB/s wr, 397 op/s; 0 B/s, 5 objects/s recovering 2024-09-21T12:32:22.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:22 smithi148 ceph-mon[49387]: pgmap v88: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.6 MiB/s wr, 397 op/s; 0 B/s, 5 objects/s recovering 2024-09-21T12:32:22.820 INFO:teuthology.orchestra.run.smithi078.stdout: 5 170578 9.98 MB/sec execute 300 sec latency 62.143 ms 2024-09-21T12:32:23.629 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:23 smithi078 ceph-mon[63866]: Upgrade: Updating osd.1 2024-09-21T12:32:23.629 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:23 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:23.629 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:23 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T12:32:23.629 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:23 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:32:23.629 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:23 smithi078 ceph-mon[63866]: Deploying daemon osd.1 on smithi078 2024-09-21T12:32:23.820 INFO:teuthology.orchestra.run.smithi078.stdout: 5 170843 9.99 MB/sec execute 301 sec latency 53.769 ms 2024-09-21T12:32:24.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:23 smithi148 ceph-mon[49387]: Upgrade: Updating osd.1 2024-09-21T12:32:24.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:23 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:24.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:23 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T12:32:24.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:23 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:32:24.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:23 smithi148 ceph-mon[49387]: Deploying daemon osd.1 on smithi078 2024-09-21T12:32:24.638 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:24 smithi078 ceph-mon[63866]: pgmap v89: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.6 MiB/s wr, 376 op/s; 0 B/s, 5 objects/s recovering 2024-09-21T12:32:24.821 INFO:teuthology.orchestra.run.smithi078.stdout: 5 171091 9.97 MB/sec execute 302 sec latency 53.080 ms 2024-09-21T12:32:25.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:24 smithi148 ceph-mon[49387]: pgmap v89: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.6 MiB/s wr, 376 op/s; 0 B/s, 5 objects/s recovering 2024-09-21T12:32:25.820 INFO:teuthology.orchestra.run.smithi078.stdout: 5 171360 9.96 MB/sec execute 303 sec latency 68.466 ms 2024-09-21T12:32:25.902 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:25 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:25.903 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:25 smithi078 ceph-mon[63866]: pgmap v90: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.2 MiB/s wr, 331 op/s; 0 B/s, 4 objects/s recovering 2024-09-21T12:32:26.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:25 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:26.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:25 smithi148 ceph-mon[49387]: pgmap v90: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.2 MiB/s wr, 331 op/s; 0 B/s, 4 objects/s recovering 2024-09-21T12:32:26.821 INFO:teuthology.orchestra.run.smithi078.stdout: 5 171619 9.93 MB/sec execute 304 sec latency 74.369 ms 2024-09-21T12:32:27.172 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:27 smithi078 systemd[1]: Stopping Ceph osd.1 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:32:27.374 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-21T12:32:27.680 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:27 smithi078 ceph-mon[63866]: osd.1 marked itself down and dead 2024-09-21T12:32:27.680 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:27 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1[38191]: 2024-09-21T12:32:27.485+0000 7f6bfad0a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T12:32:27.680 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:27 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1[38191]: 2024-09-21T12:32:27.485+0000 7f6bfad0a700 -1 osd.1 86 *** Got signal Terminated *** 2024-09-21T12:32:27.680 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:27 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1[38191]: 2024-09-21T12:32:27.485+0000 7f6bfad0a700 -1 osd.1 86 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T12:32:27.821 INFO:teuthology.orchestra.run.smithi078.stdout: 5 171854 9.92 MB/sec execute 305 sec latency 253.378 ms 2024-09-21T12:32:28.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:27 smithi148 ceph-mon[49387]: osd.1 marked itself down and dead 2024-09-21T12:32:28.217 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T12:32:28.217 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (4m) 29s ago 13m 19.3M - 0.27.0 11f11916f8cd e6e67ab07dec 2024-09-21T12:32:28.217 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (80s) 29s ago 13m 6895k - 19.3.0-5104-g44178494 b415816db8b4 46a10ea0b111 2024-09-21T12:32:28.217 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi148 smithi148 running (75s) 70s ago 13m 6895k - 19.3.0-5104-g44178494 b415816db8b4 23deb0615621 2024-09-21T12:32:28.217 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (4m) 29s ago 13m 52.3M - dad864ee21e9 2a140c36091b 2024-09-21T12:32:28.217 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.ytpzwh smithi078 running (7m) 29s ago 7m 37.9M - 17.2.0 e1d6a67b021e 5e9b109502a2 2024-09-21T12:32:28.217 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi148.uukezr smithi148 running (7m) 70s ago 7m 13.7M - 17.2.0 e1d6a67b021e b7e8aa0f3a6c 2024-09-21T12:32:28.217 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.ntwyan smithi078 *:8443,9283,8765 running (3m) 29s ago 14m 511M - 19.3.0-5104-g44178494 b415816db8b4 a61baedebf43 2024-09-21T12:32:28.218 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi148.oyiqmx smithi148 *:8443,9283,8765 running (2m) 70s ago 13m 454M - 19.3.0-5104-g44178494 b415816db8b4 0255535c6fdb 2024-09-21T12:32:28.218 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (2m) 29s ago 15m 50.2M 2048M 19.3.0-5104-g44178494 b415816db8b4 22b3097efbd8 2024-09-21T12:32:28.218 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi148 smithi148 running (2m) 70s ago 12m 36.1M 2048M 19.3.0-5104-g44178494 b415816db8b4 df8b08509136 2024-09-21T12:32:28.218 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.ydbhru smithi078 *:2049 running (7m) 29s ago 7m 265M - 4.0 e1d6a67b021e 7e5b0274ac38 2024-09-21T12:32:28.218 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi148.einuzt smithi148 *:2049 running (7m) 70s ago 7m 86.8M - 4.0 e1d6a67b021e 76a8b0eabede 2024-09-21T12:32:28.218 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (4m) 29s ago 13m 11.7M - 1.7.0 72c9c2088986 301adb4e9e22 2024-09-21T12:32:28.218 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi148 smithi148 *:9100 running (3m) 70s ago 12m 11.3M - 1.7.0 72c9c2088986 0cceff4bf342 2024-09-21T12:32:28.218 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (38s) 29s ago 12m 14.5M 1936M 19.3.0-5104-g44178494 b415816db8b4 7374add97534 2024-09-21T12:32:28.218 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (11m) 29s ago 11m 482M 1936M 17.2.0 e1d6a67b021e b9a4b952233b 2024-09-21T12:32:28.218 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (11m) 29s ago 11m 422M 1936M 17.2.0 e1d6a67b021e 829fb8c2d827 2024-09-21T12:32:28.218 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (10m) 29s ago 10m 508M 1936M 17.2.0 e1d6a67b021e ef942c8556d0 2024-09-21T12:32:28.218 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi148 running (10m) 70s ago 10m 493M 2704M 17.2.0 e1d6a67b021e 62eefdd572c0 2024-09-21T12:32:28.218 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi148 running (9m) 70s ago 9m 314M 2704M 17.2.0 e1d6a67b021e 7b44be78baca 2024-09-21T12:32:28.218 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi148 running (9m) 70s ago 9m 509M 2704M 17.2.0 e1d6a67b021e 3d23057910ea 2024-09-21T12:32:28.219 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi148 running (9m) 70s ago 9m 334M 2704M 17.2.0 e1d6a67b021e f9e4db9d35fa 2024-09-21T12:32:28.219 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (3m) 29s ago 12m 49.2M - 2.51.0 1d3b7f56885b face2d4cdd1b 2024-09-21T12:32:28.493 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:28 smithi078 podman[70606]: 2024-09-21 12:32:28.209686995 +0000 UTC m=+0.880529126 container died b9a4b952233b091df3c9d54cd2359b8c0da8859f21c48d767a52935fc591597d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1, name=centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., architecture=x86_64, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, ceph=True, GIT_BRANCH=HEAD, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, release=754, RELEASE=HEAD, io.buildah.version=1.19.8) 2024-09-21T12:32:28.730 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:32:28.730 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-21T12:32:28.730 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:32:28.731 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:32:28.731 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-21T12:32:28.731 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:32:28.731 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:32:28.731 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-21T12:32:28.731 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-21T12:32:28.731 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 1 2024-09-21T12:32:28.732 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:32:28.732 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-21T12:32:28.732 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:32:28.732 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:32:28.732 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-21T12:32:28.733 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-09-21T12:32:28.733 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 5 2024-09-21T12:32:28.733 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T12:32:28.733 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:32:28.742 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:28 smithi078 ceph-mon[63866]: from='client.44161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:32:28.743 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:28 smithi078 ceph-mon[63866]: pgmap v91: 97 pgs: 97 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.0 MiB/s wr, 506 op/s; 0 B/s, 4 objects/s recovering 2024-09-21T12:32:28.743 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:28 smithi078 ceph-mon[63866]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T12:32:28.743 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:28 smithi078 ceph-mon[63866]: osdmap e87: 8 total, 7 up, 8 in 2024-09-21T12:32:28.743 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:28 smithi078 ceph-mon[63866]: osdmap e88: 8 total, 7 up, 8 in 2024-09-21T12:32:28.743 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:28 smithi078 podman[70606]: 2024-09-21 12:32:28.615042576 +0000 UTC m=+1.285884693 container cleanup b9a4b952233b091df3c9d54cd2359b8c0da8859f21c48d767a52935fc591597d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1, io.buildah.version=1.19.8, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, com.redhat.component=centos-stream-container, GIT_CLEAN=True, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., architecture=x86_64, ceph=True, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-21T12:32:28.743 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:28 smithi078 bash[70606]: ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1 2024-09-21T12:32:28.821 INFO:teuthology.orchestra.run.smithi078.stdout: 5 171920 9.89 MB/sec execute 306 sec latency 1005.919 ms 2024-09-21T12:32:29.011 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:28 smithi078 podman[70668]: 2024-09-21 12:32:28.930785021 +0000 UTC m=+0.720989721 container remove b9a4b952233b091df3c9d54cd2359b8c0da8859f21c48d767a52935fc591597d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, version=8, io.openshift.tags=base centos centos-stream, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, vcs-type=git, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-21T12:32:29.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:28 smithi148 ceph-mon[49387]: from='client.44161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:32:29.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:28 smithi148 ceph-mon[49387]: pgmap v91: 97 pgs: 97 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.0 MiB/s wr, 506 op/s; 0 B/s, 4 objects/s recovering 2024-09-21T12:32:29.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:28 smithi148 ceph-mon[49387]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T12:32:29.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:28 smithi148 ceph-mon[49387]: osdmap e87: 8 total, 7 up, 8 in 2024-09-21T12:32:29.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:28 smithi148 ceph-mon[49387]: osdmap e88: 8 total, 7 up, 8 in 2024-09-21T12:32:29.235 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:32:29.236 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5", 2024-09-21T12:32:29.236 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-21T12:32:29.236 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T12:32:29.236 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-21T12:32:29.236 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-21T12:32:29.236 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-09-21T12:32:29.236 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr" 2024-09-21T12:32:29.236 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-21T12:32:29.236 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "7/23 daemons upgraded", 2024-09-21T12:32:29.236 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T12:32:29.236 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-21T12:32:29.236 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:32:29.772 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:29 smithi078 ceph-mon[63866]: from='client.34228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:32:29.772 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:29 smithi078 ceph-mon[63866]: from='client.34232 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:32:29.773 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:29 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/3161731921' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:32:29.773 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:29 smithi078 podman[70820]: 2024-09-21 12:32:29.445639343 +0000 UTC m=+0.024884283 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:32:29.773 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:29 smithi078 podman[70820]: 2024-09-21 12:32:29.596534159 +0000 UTC m=+0.175779086 container create 74c1af8e97bc5e787e73e76264b2ab9777db87c29f0602634c74acbaf2266d6c (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-21T12:32:29.821 INFO:teuthology.orchestra.run.smithi078.stdout: 5 172101 9.88 MB/sec execute 307 sec latency 92.583 ms 2024-09-21T12:32:29.876 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_WARN 1 osds down 2024-09-21T12:32:29.876 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-21T12:32:29.876 INFO:teuthology.orchestra.run.smithi078.stdout: osd.1 (root=default,host=smithi078) is down 2024-09-21T12:32:30.036 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:29 smithi078 podman[70820]: 2024-09-21 12:32:29.967253153 +0000 UTC m=+0.546498088 container init 74c1af8e97bc5e787e73e76264b2ab9777db87c29f0602634c74acbaf2266d6c (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-21T12:32:30.036 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:29 smithi078 podman[70820]: 2024-09-21 12:32:29.972385442 +0000 UTC m=+0.551630368 container start 74c1af8e97bc5e787e73e76264b2ab9777db87c29f0602634c74acbaf2266d6c (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-21T12:32:30.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:29 smithi148 ceph-mon[49387]: from='client.34228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:32:30.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:29 smithi148 ceph-mon[49387]: from='client.34232 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:32:30.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:29 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/3161731921' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:32:30.287 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:30 smithi078 podman[70820]: 2024-09-21 12:32:30.101030756 +0000 UTC m=+0.680275683 container attach 74c1af8e97bc5e787e73e76264b2ab9777db87c29f0602634c74acbaf2266d6c (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1-deactivate, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T12:32:30.538 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:30 smithi078 podman[70820]: 2024-09-21 12:32:30.297403834 +0000 UTC m=+0.876648776 container died 74c1af8e97bc5e787e73e76264b2ab9777db87c29f0602634c74acbaf2266d6c (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1-deactivate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True) 2024-09-21T12:32:30.788 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:30 smithi078 ceph-mon[63866]: from='client.34240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:32:30.789 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:30 smithi078 ceph-mon[63866]: pgmap v94: 97 pgs: 14 stale+active+clean, 83 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.2 MiB/s wr, 284 op/s 2024-09-21T12:32:30.789 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:30.789 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:32:30.789 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:30 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/2941898381' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:32:30.821 INFO:teuthology.orchestra.run.smithi078.stdout: 5 172531 9.89 MB/sec execute 308 sec latency 87.480 ms 2024-09-21T12:32:31.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:30 smithi148 ceph-mon[49387]: from='client.34240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:32:31.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:30 smithi148 ceph-mon[49387]: pgmap v94: 97 pgs: 14 stale+active+clean, 83 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.2 MiB/s wr, 284 op/s 2024-09-21T12:32:31.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:31.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:32:31.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:30 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/2941898381' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:32:31.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:31 smithi078 ceph-mon[63866]: Health check failed: Degraded data redundancy: 154/1542 objects degraded (9.987%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T12:32:31.821 INFO:teuthology.orchestra.run.smithi078.stdout: 5 172990 9.89 MB/sec execute 309 sec latency 73.560 ms 2024-09-21T12:32:32.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:31 smithi148 ceph-mon[49387]: Health check failed: Degraded data redundancy: 154/1542 objects degraded (9.987%), 22 pgs degraded (PG_DEGRADED) 2024-09-21T12:32:32.329 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:32 smithi078 podman[70846]: 2024-09-21 12:32:32.153179792 +0000 UTC m=+1.847264461 container remove 74c1af8e97bc5e787e73e76264b2ab9777db87c29f0602634c74acbaf2266d6c (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0842-4417849, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-21T12:32:32.329 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:32 smithi078 systemd[1]: ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.1.service: Deactivated successfully. 2024-09-21T12:32:32.329 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:32 smithi078 systemd[1]: Stopped Ceph osd.1 for 6263deca-7813-11ef-baf6-efdc52797490. 2024-09-21T12:32:32.329 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:32 smithi078 systemd[1]: ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.1.service: Consumed 2min 57.133s CPU time. 2024-09-21T12:32:32.589 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:32 smithi078 systemd[1]: Starting Ceph osd.1 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:32:32.821 INFO:teuthology.orchestra.run.smithi078.stdout: 5 173309 9.89 MB/sec execute 310 sec latency 81.063 ms 2024-09-21T12:32:32.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:32 smithi078 ceph-mon[63866]: pgmap v95: 97 pgs: 13 active+undersized, 1 stale+active+clean, 22 active+undersized+degraded, 61 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.5 MiB/s wr, 453 op/s; 154/1542 objects degraded (9.987%) 2024-09-21T12:32:32.897 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:32 smithi078 podman[70957]: 2024-09-21 12:32:32.662850495 +0000 UTC m=+0.021236572 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:32:33.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:32 smithi148 ceph-mon[49387]: pgmap v95: 97 pgs: 13 active+undersized, 1 stale+active+clean, 22 active+undersized+degraded, 61 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.5 MiB/s wr, 453 op/s; 154/1542 objects degraded (9.987%) 2024-09-21T12:32:33.147 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:33 smithi078 podman[70957]: 2024-09-21 12:32:33.029849333 +0000 UTC m=+0.388235393 container create 063e1ccb753ab627cc369841333ae4c98afd8caddf1c1b1648c32be26c5e8785 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1-activate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T12:32:33.412 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:33 smithi078 podman[70957]: 2024-09-21 12:32:33.38387982 +0000 UTC m=+0.742265898 container init 063e1ccb753ab627cc369841333ae4c98afd8caddf1c1b1648c32be26c5e8785 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1-activate, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-21T12:32:33.412 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:33 smithi078 podman[70957]: 2024-09-21 12:32:33.390004351 +0000 UTC m=+0.748390409 container start 063e1ccb753ab627cc369841333ae4c98afd8caddf1c1b1648c32be26c5e8785 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-21-0842-4417849, CEPH_POINT_RELEASE=, ceph=True) 2024-09-21T12:32:33.666 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:33 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1-activate[70971]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:32:33.666 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:33 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1-activate[70971]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:32:33.666 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:33 smithi078 bash[70957]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:32:33.666 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:33 smithi078 bash[70957]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:32:33.666 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:33 smithi078 podman[70957]: 2024-09-21 12:32:33.568002054 +0000 UTC m=+0.926388120 container attach 063e1ccb753ab627cc369841333ae4c98afd8caddf1c1b1648c32be26c5e8785 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1-activate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_BRANCH=HEAD) 2024-09-21T12:32:33.821 INFO:teuthology.orchestra.run.smithi078.stdout: 5 173596 9.88 MB/sec execute 311 sec latency 124.426 ms 2024-09-21T12:32:34.670 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:34 smithi078 ceph-mon[63866]: pgmap v96: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.0 MiB/s wr, 529 op/s; 170/1518 objects degraded (11.199%) 2024-09-21T12:32:34.821 INFO:teuthology.orchestra.run.smithi078.stdout: 5 174076 9.90 MB/sec execute 312 sec latency 63.969 ms 2024-09-21T12:32:35.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:34 smithi148 ceph-mon[49387]: pgmap v96: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.0 MiB/s wr, 529 op/s; 170/1518 objects degraded (11.199%) 2024-09-21T12:32:35.822 INFO:teuthology.orchestra.run.smithi078.stdout: 5 174559 9.92 MB/sec execute 313 sec latency 33.188 ms 2024-09-21T12:32:36.713 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:36 smithi078 ceph-mon[63866]: pgmap v97: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 2.2 MiB/s wr, 266 op/s; 170/1518 objects degraded (11.199%) 2024-09-21T12:32:36.821 INFO:teuthology.orchestra.run.smithi078.stdout: 5 174988 9.91 MB/sec execute 314 sec latency 42.798 ms 2024-09-21T12:32:37.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:36 smithi148 ceph-mon[49387]: pgmap v97: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 2.2 MiB/s wr, 266 op/s; 170/1518 objects degraded (11.199%) 2024-09-21T12:32:37.762 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:37 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 170/1518 objects degraded (11.199%), 24 pgs degraded (PG_DEGRADED) 2024-09-21T12:32:37.821 INFO:teuthology.orchestra.run.smithi078.stdout: 5 175392 9.91 MB/sec execute 315 sec latency 43.561 ms 2024-09-21T12:32:38.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:37 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 170/1518 objects degraded (11.199%), 24 pgs degraded (PG_DEGRADED) 2024-09-21T12:32:38.271 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:38 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1-activate[70971]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T12:32:38.271 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:38 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1-activate[70971]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:32:38.271 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:38 smithi078 bash[70957]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T12:32:38.271 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:38 smithi078 bash[70957]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:32:38.271 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:38 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1-activate[70971]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:32:38.271 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:38 smithi078 bash[70957]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:32:38.272 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:38 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1-activate[70971]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T12:32:38.272 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:38 smithi078 bash[70957]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T12:32:38.272 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:38 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1-activate[70971]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2024-09-21T12:32:38.272 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:38 smithi078 bash[70957]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2024-09-21T12:32:38.773 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:38 smithi078 ceph-mon[63866]: pgmap v98: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.8 MiB/s wr, 486 op/s; 188/1728 objects degraded (10.880%) 2024-09-21T12:32:38.774 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:38 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1-activate[70971]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-21T12:32:38.774 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:38 smithi078 bash[70957]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-21T12:32:38.774 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:38 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1-activate[70971]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2024-09-21T12:32:38.774 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:38 smithi078 bash[70957]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2024-09-21T12:32:38.774 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:38 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1-activate[70971]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T12:32:38.774 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:38 smithi078 bash[70957]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T12:32:38.774 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:38 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1-activate[70971]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T12:32:38.774 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:38 smithi078 bash[70957]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T12:32:38.775 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:38 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1-activate[70971]: --> ceph-volume lvm activate successful for osd ID: 1 2024-09-21T12:32:38.775 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:38 smithi078 bash[70957]: --> ceph-volume lvm activate successful for osd ID: 1 2024-09-21T12:32:38.775 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:38 smithi078 podman[70957]: 2024-09-21 12:32:38.652803017 +0000 UTC m=+6.011189080 container died 063e1ccb753ab627cc369841333ae4c98afd8caddf1c1b1648c32be26c5e8785 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1-activate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0842-4417849, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-21T12:32:38.821 INFO:teuthology.orchestra.run.smithi078.stdout: 5 175673 9.90 MB/sec execute 316 sec latency 87.165 ms 2024-09-21T12:32:39.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:38 smithi148 ceph-mon[49387]: pgmap v98: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.8 MiB/s wr, 486 op/s; 188/1728 objects degraded (10.880%) 2024-09-21T12:32:39.794 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:39 smithi078 podman[71200]: 2024-09-21 12:32:39.543157614 +0000 UTC m=+0.872164911 container remove 063e1ccb753ab627cc369841333ae4c98afd8caddf1c1b1648c32be26c5e8785 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1-activate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-21-0842-4417849, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD) 2024-09-21T12:32:39.821 INFO:teuthology.orchestra.run.smithi078.stdout: 5 176088 9.91 MB/sec execute 317 sec latency 59.215 ms 2024-09-21T12:32:40.052 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:39 smithi078 podman[71258]: 2024-09-21 12:32:39.701752787 +0000 UTC m=+0.025977270 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:32:40.052 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:39 smithi078 podman[71258]: 2024-09-21 12:32:39.853236448 +0000 UTC m=+0.177460911 container create 14655a8da4e56dfa60e289d895691a7ad0a0918e65acfcd8a22bfeadc4e1240e (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-21T12:32:40.305 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:40 smithi078 podman[71258]: 2024-09-21 12:32:40.190441817 +0000 UTC m=+0.514666291 container init 14655a8da4e56dfa60e289d895691a7ad0a0918e65acfcd8a22bfeadc4e1240e (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T12:32:40.306 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:40 smithi078 podman[71258]: 2024-09-21 12:32:40.194856157 +0000 UTC m=+0.519080626 container start 14655a8da4e56dfa60e289d895691a7ad0a0918e65acfcd8a22bfeadc4e1240e (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T12:32:40.306 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:40 smithi078 ceph-osd[71275]: -- 172.21.15.78:0/1875049027 <== mon.0 v2:172.21.15.78:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55646959c1e0 con 0x556469528000 2024-09-21T12:32:40.306 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:40 smithi078 bash[71258]: 14655a8da4e56dfa60e289d895691a7ad0a0918e65acfcd8a22bfeadc4e1240e 2024-09-21T12:32:40.306 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:40 smithi078 systemd[1]: Started Ceph osd.1 for 6263deca-7813-11ef-baf6-efdc52797490. 2024-09-21T12:32:40.807 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:40 smithi078 ceph-mon[63866]: pgmap v99: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.4 MiB/s wr, 442 op/s; 188/1728 objects degraded (10.880%) 2024-09-21T12:32:40.808 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:40 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:40.808 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:40 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:40.808 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:40 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:32:40.821 INFO:teuthology.orchestra.run.smithi078.stdout: 5 176404 9.91 MB/sec execute 318 sec latency 39.449 ms 2024-09-21T12:32:41.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:40 smithi148 ceph-mon[49387]: pgmap v99: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.4 MiB/s wr, 442 op/s; 188/1728 objects degraded (10.880%) 2024-09-21T12:32:41.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:40 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:41.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:40 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:41.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:40 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:32:41.822 INFO:teuthology.orchestra.run.smithi078.stdout: 5 176841 9.91 MB/sec execute 319 sec latency 73.529 ms 2024-09-21T12:32:42.326 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:42 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1[71271]: 2024-09-21T12:32:42.259+0000 7fd06a444640 -1 Falling back to public interface 2024-09-21T12:32:42.822 INFO:teuthology.orchestra.run.smithi078.stdout: 5 177084 9.89 MB/sec execute 320 sec latency 54.096 ms 2024-09-21T12:32:42.835 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:42 smithi078 ceph-mon[63866]: pgmap v100: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.5 MiB/s wr, 540 op/s; 182/1626 objects degraded (11.193%) 2024-09-21T12:32:42.836 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:42 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 182/1626 objects degraded (11.193%), 24 pgs degraded (PG_DEGRADED) 2024-09-21T12:32:43.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:42 smithi148 ceph-mon[49387]: pgmap v100: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.5 MiB/s wr, 540 op/s; 182/1626 objects degraded (11.193%) 2024-09-21T12:32:43.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:42 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 182/1626 objects degraded (11.193%), 24 pgs degraded (PG_DEGRADED) 2024-09-21T12:32:43.822 INFO:teuthology.orchestra.run.smithi078.stdout: 5 177395 9.91 MB/sec execute 321 sec latency 103.626 ms 2024-09-21T12:32:44.822 INFO:teuthology.orchestra.run.smithi078.stdout: 5 177726 9.90 MB/sec execute 322 sec latency 50.380 ms 2024-09-21T12:32:44.870 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:44 smithi078 ceph-mon[63866]: pgmap v101: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.0 MiB/s wr, 489 op/s; 180/1602 objects degraded (11.236%) 2024-09-21T12:32:45.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:44 smithi148 ceph-mon[49387]: pgmap v101: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.0 MiB/s wr, 489 op/s; 180/1602 objects degraded (11.236%) 2024-09-21T12:32:45.822 INFO:teuthology.orchestra.run.smithi078.stdout: 5 178056 9.90 MB/sec execute 323 sec latency 53.975 ms 2024-09-21T12:32:45.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:45.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:32:45.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:45 smithi078 ceph-mon[63866]: pgmap v102: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.6 MiB/s wr, 438 op/s; 180/1602 objects degraded (11.236%) 2024-09-21T12:32:46.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:46.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:32:46.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:45 smithi148 ceph-mon[49387]: pgmap v102: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.6 MiB/s wr, 438 op/s; 180/1602 objects degraded (11.236%) 2024-09-21T12:32:46.822 INFO:teuthology.orchestra.run.smithi078.stdout: 5 178293 9.89 MB/sec execute 324 sec latency 58.497 ms 2024-09-21T12:32:47.233 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:46 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 180/1602 objects degraded (11.236%), 24 pgs degraded (PG_DEGRADED) 2024-09-21T12:32:47.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:46 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 180/1602 objects degraded (11.236%), 24 pgs degraded (PG_DEGRADED) 2024-09-21T12:32:47.822 INFO:teuthology.orchestra.run.smithi078.stdout: 5 178595 9.88 MB/sec execute 325 sec latency 63.842 ms 2024-09-21T12:32:48.243 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:47 smithi078 ceph-mon[63866]: pgmap v103: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.7 MiB/s wr, 632 op/s; 189/1686 objects degraded (11.210%) 2024-09-21T12:32:48.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:47 smithi148 ceph-mon[49387]: pgmap v103: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.7 MiB/s wr, 632 op/s; 189/1686 objects degraded (11.210%) 2024-09-21T12:32:48.822 INFO:teuthology.orchestra.run.smithi078.stdout: 5 178827 9.86 MB/sec execute 326 sec latency 64.116 ms 2024-09-21T12:32:49.007 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:48 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1[71271]: 2024-09-21T12:32:48.925+0000 7fd06a444640 -1 osd.1 0 read_superblock omap replica is missing. 2024-09-21T12:32:49.822 INFO:teuthology.orchestra.run.smithi078.stdout: 5 179063 9.85 MB/sec execute 327 sec latency 105.589 ms 2024-09-21T12:32:50.054 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:49 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1[71271]: 2024-09-21T12:32:49.837+0000 7fd06a444640 -1 osd.1 86 log_to_monitors true 2024-09-21T12:32:50.822 INFO:teuthology.orchestra.run.smithi078.stdout: 5 179488 9.86 MB/sec execute 328 sec latency 88.516 ms 2024-09-21T12:32:50.837 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:50 smithi078 ceph-mon[63866]: pgmap v104: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.2 MiB/s wr, 407 op/s; 189/1686 objects degraded (11.210%) 2024-09-21T12:32:50.837 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:50 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:50.837 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:50 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:50.837 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:50 smithi078 ceph-mon[63866]: from='osd.1 [v2:172.21.15.78:6810/4007548121,v1:172.21.15.78:6811/4007548121]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T12:32:50.837 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:50 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:50.837 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:50 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:51.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:50 smithi148 ceph-mon[49387]: pgmap v104: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.2 MiB/s wr, 407 op/s; 189/1686 objects degraded (11.210%) 2024-09-21T12:32:51.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:50 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:51.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:50 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:51.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:50 smithi148 ceph-mon[49387]: from='osd.1 [v2:172.21.15.78:6810/4007548121,v1:172.21.15.78:6811/4007548121]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T12:32:51.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:50 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:51.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:50 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:51.345 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:32:51 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1[71271]: 2024-09-21T12:32:51.276+0000 7fd05e1e6640 -1 osd.1 86 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T12:32:51.822 INFO:teuthology.orchestra.run.smithi078.stdout: 5 179858 9.86 MB/sec execute 329 sec latency 99.906 ms 2024-09-21T12:32:51.924 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:51 smithi078 ceph-mon[63866]: from='osd.1 [v2:172.21.15.78:6810/4007548121,v1:172.21.15.78:6811/4007548121]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T12:32:51.924 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:51 smithi078 ceph-mon[63866]: osdmap e89: 8 total, 7 up, 8 in 2024-09-21T12:32:51.924 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:51 smithi078 ceph-mon[63866]: from='osd.1 [v2:172.21.15.78:6810/4007548121,v1:172.21.15.78:6811/4007548121]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]} : dispatch 2024-09-21T12:32:51.924 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:51 smithi078 ceph-mon[63866]: from='osd.1 [v2:172.21.15.78:6810/4007548121,v1:172.21.15.78:6811/4007548121]' entity='osd.1' 2024-09-21T12:32:52.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:51 smithi148 ceph-mon[49387]: from='osd.1 [v2:172.21.15.78:6810/4007548121,v1:172.21.15.78:6811/4007548121]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T12:32:52.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:51 smithi148 ceph-mon[49387]: osdmap e89: 8 total, 7 up, 8 in 2024-09-21T12:32:52.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:51 smithi148 ceph-mon[49387]: from='osd.1 [v2:172.21.15.78:6810/4007548121,v1:172.21.15.78:6811/4007548121]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi078", "root=default"]} : dispatch 2024-09-21T12:32:52.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:51 smithi148 ceph-mon[49387]: from='osd.1 [v2:172.21.15.78:6810/4007548121,v1:172.21.15.78:6811/4007548121]' entity='osd.1' 2024-09-21T12:32:52.822 INFO:teuthology.orchestra.run.smithi078.stdout: 5 180016 9.84 MB/sec execute 330 sec latency 581.618 ms 2024-09-21T12:32:52.968 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:52 smithi078 ceph-mon[63866]: pgmap v106: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.0 MiB/s wr, 525 op/s; 178/1647 objects degraded (10.808%) 2024-09-21T12:32:52.968 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:52 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 189/1686 objects degraded (11.210%), 24 pgs degraded (PG_DEGRADED) 2024-09-21T12:32:52.968 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:52 smithi078 ceph-mon[63866]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T12:32:52.968 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:52 smithi078 ceph-mon[63866]: osd.1 [v2:172.21.15.78:6810/4007548121,v1:172.21.15.78:6811/4007548121] boot 2024-09-21T12:32:52.968 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:52 smithi078 ceph-mon[63866]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T12:32:52.968 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T12:32:53.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:52 smithi148 ceph-mon[49387]: pgmap v106: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.0 MiB/s wr, 525 op/s; 178/1647 objects degraded (10.808%) 2024-09-21T12:32:53.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:52 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 189/1686 objects degraded (11.210%), 24 pgs degraded (PG_DEGRADED) 2024-09-21T12:32:53.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:52 smithi148 ceph-mon[49387]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T12:32:53.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:52 smithi148 ceph-mon[49387]: osd.1 [v2:172.21.15.78:6810/4007548121,v1:172.21.15.78:6811/4007548121] boot 2024-09-21T12:32:53.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:52 smithi148 ceph-mon[49387]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T12:32:53.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T12:32:53.822 INFO:teuthology.orchestra.run.smithi078.stdout: 5 180016 9.81 MB/sec execute 331 sec latency 1581.642 ms 2024-09-21T12:32:54.424 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:54 smithi078 ceph-mon[63866]: osdmap e91: 8 total, 8 up, 8 in 2024-09-21T12:32:54.424 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:54 smithi078 ceph-mon[63866]: pgmap v109: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.5 MiB/s wr, 362 op/s; 181/1662 objects degraded (10.890%) 2024-09-21T12:32:54.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:54 smithi148 ceph-mon[49387]: osdmap e91: 8 total, 8 up, 8 in 2024-09-21T12:32:54.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:54 smithi148 ceph-mon[49387]: pgmap v109: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.5 MiB/s wr, 362 op/s; 181/1662 objects degraded (10.890%) 2024-09-21T12:32:54.822 INFO:teuthology.orchestra.run.smithi078.stdout: 5 180267 9.80 MB/sec execute 332 sec latency 2181.759 ms 2024-09-21T12:32:55.434 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:55 smithi078 ceph-mon[63866]: osdmap e92: 8 total, 8 up, 8 in 2024-09-21T12:32:55.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:55 smithi148 ceph-mon[49387]: osdmap e92: 8 total, 8 up, 8 in 2024-09-21T12:32:55.822 INFO:teuthology.orchestra.run.smithi078.stdout: 5 180604 9.79 MB/sec execute 333 sec latency 122.222 ms 2024-09-21T12:32:56.449 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:56 smithi078 ceph-mon[63866]: osdmap e93: 8 total, 8 up, 8 in 2024-09-21T12:32:56.449 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:56 smithi078 ceph-mon[63866]: pgmap v112: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 853 KiB/s wr, 38 op/s; 181/1662 objects degraded (10.890%) 2024-09-21T12:32:56.449 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:56 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:56.449 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:56 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:56.449 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:56 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:32:56.449 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:56 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:32:56.449 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:56 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:56.449 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:56 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:56.449 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:56 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:32:56.449 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:56 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:32:56.450 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:56 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:32:56.450 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:56 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:32:56.450 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:56 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T12:32:56.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:56 smithi148 ceph-mon[49387]: osdmap e93: 8 total, 8 up, 8 in 2024-09-21T12:32:56.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:56 smithi148 ceph-mon[49387]: pgmap v112: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 853 KiB/s wr, 38 op/s; 181/1662 objects degraded (10.890%) 2024-09-21T12:32:56.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:56 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:56.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:56 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:56.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:56 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:32:56.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:56 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:32:56.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:56 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:56.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:56 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:32:56.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:56 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:32:56.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:56 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:32:56.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:56 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:32:56.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:56 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:32:56.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:56 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T12:32:56.822 INFO:teuthology.orchestra.run.smithi078.stdout: 5 180815 9.79 MB/sec execute 334 sec latency 580.118 ms 2024-09-21T12:32:57.468 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:57 smithi078 ceph-mon[63866]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T12:32:57.468 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:57 smithi078 ceph-mon[63866]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-09-21T12:32:57.468 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:57 smithi078 ceph-mon[63866]: osdmap e94: 8 total, 8 up, 8 in 2024-09-21T12:32:57.468 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:57 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 181/1662 objects degraded (10.890%), 24 pgs degraded (PG_DEGRADED) 2024-09-21T12:32:57.468 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:57 smithi078 ceph-mon[63866]: osdmap e95: 8 total, 8 up, 8 in 2024-09-21T12:32:57.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:57 smithi148 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T12:32:57.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:57 smithi148 ceph-mon[49387]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-09-21T12:32:57.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:57 smithi148 ceph-mon[49387]: osdmap e94: 8 total, 8 up, 8 in 2024-09-21T12:32:57.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:57 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 181/1662 objects degraded (10.890%), 24 pgs degraded (PG_DEGRADED) 2024-09-21T12:32:57.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:57 smithi148 ceph-mon[49387]: osdmap e95: 8 total, 8 up, 8 in 2024-09-21T12:32:57.822 INFO:teuthology.orchestra.run.smithi078.stdout: 5 181044 9.77 MB/sec execute 335 sec latency 776.372 ms 2024-09-21T12:32:58.338 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:58 smithi078 ceph-mon[63866]: pgmap v115: 97 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 4 peering, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.9 MiB/s wr, 408 op/s; 228/1617 objects degraded (14.100%); 1.0 MiB/s, 52 keys/s, 20 objects/s recovering 2024-09-21T12:32:58.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:58 smithi148 ceph-mon[49387]: pgmap v115: 97 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 4 peering, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.9 MiB/s wr, 408 op/s; 228/1617 objects degraded (14.100%); 1.0 MiB/s, 52 keys/s, 20 objects/s recovering 2024-09-21T12:32:58.822 INFO:teuthology.orchestra.run.smithi078.stdout: 5 181426 9.78 MB/sec execute 336 sec latency 910.168 ms 2024-09-21T12:32:59.549 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:59 smithi078 ceph-mon[63866]: osdmap e96: 8 total, 8 up, 8 in 2024-09-21T12:32:59.549 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:32:59 smithi078 ceph-mon[63866]: osdmap e97: 8 total, 8 up, 8 in 2024-09-21T12:32:59.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:59 smithi148 ceph-mon[49387]: osdmap e96: 8 total, 8 up, 8 in 2024-09-21T12:32:59.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:32:59 smithi148 ceph-mon[49387]: osdmap e97: 8 total, 8 up, 8 in 2024-09-21T12:32:59.822 INFO:teuthology.orchestra.run.smithi078.stdout: 5 181643 9.77 MB/sec execute 337 sec latency 953.642 ms 2024-09-21T12:33:00.304 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:00 smithi078 ceph-mon[63866]: pgmap v118: 97 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 4 peering, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.9 MiB/s wr, 408 op/s; 228/1617 objects degraded (14.100%); 1.0 MiB/s, 52 keys/s, 20 objects/s recovering 2024-09-21T12:33:00.304 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:00.304 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:33:00.304 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:00 smithi078 ceph-mon[63866]: osdmap e98: 8 total, 8 up, 8 in 2024-09-21T12:33:00.367 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-21T12:33:00.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:00 smithi148 ceph-mon[49387]: pgmap v118: 97 pgs: 8 active+recovery_wait+undersized+degraded+remapped, 4 peering, 1 active+recovering+undersized+degraded+remapped, 84 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.9 MiB/s wr, 408 op/s; 228/1617 objects degraded (14.100%); 1.0 MiB/s, 52 keys/s, 20 objects/s recovering 2024-09-21T12:33:00.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:00.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:33:00.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:00 smithi148 ceph-mon[49387]: osdmap e98: 8 total, 8 up, 8 in 2024-09-21T12:33:00.822 INFO:teuthology.orchestra.run.smithi078.stdout: 5 181895 9.77 MB/sec execute 338 sec latency 53.945 ms 2024-09-21T12:33:01.292 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T12:33:01.292 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (5m) 11s ago 14m 19.3M - 0.27.0 11f11916f8cd e6e67ab07dec 2024-09-21T12:33:01.292 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (113s) 11s ago 14m 6895k - 19.3.0-5104-g44178494 b415816db8b4 46a10ea0b111 2024-09-21T12:33:01.292 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi148 smithi148 running (108s) 103s ago 13m 6895k - 19.3.0-5104-g44178494 b415816db8b4 23deb0615621 2024-09-21T12:33:01.292 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (5m) 11s ago 13m 53.4M - dad864ee21e9 2a140c36091b 2024-09-21T12:33:01.292 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.ytpzwh smithi078 running (8m) 11s ago 8m 37.8M - 17.2.0 e1d6a67b021e 5e9b109502a2 2024-09-21T12:33:01.293 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi148.uukezr smithi148 running (8m) 103s ago 8m 13.7M - 17.2.0 e1d6a67b021e b7e8aa0f3a6c 2024-09-21T12:33:01.293 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.ntwyan smithi078 *:8443,9283,8765 running (4m) 11s ago 15m 516M - 19.3.0-5104-g44178494 b415816db8b4 a61baedebf43 2024-09-21T12:33:01.293 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi148.oyiqmx smithi148 *:8443,9283,8765 running (3m) 103s ago 13m 454M - 19.3.0-5104-g44178494 b415816db8b4 0255535c6fdb 2024-09-21T12:33:01.293 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (3m) 11s ago 15m 53.2M 2048M 19.3.0-5104-g44178494 b415816db8b4 22b3097efbd8 2024-09-21T12:33:01.293 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi148 smithi148 running (2m) 103s ago 13m 36.1M 2048M 19.3.0-5104-g44178494 b415816db8b4 df8b08509136 2024-09-21T12:33:01.293 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.ydbhru smithi078 *:2049 running (7m) 11s ago 7m 278M - 4.0 e1d6a67b021e 7e5b0274ac38 2024-09-21T12:33:01.293 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi148.einuzt smithi148 *:2049 running (7m) 103s ago 7m 86.8M - 4.0 e1d6a67b021e 76a8b0eabede 2024-09-21T12:33:01.293 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (4m) 11s ago 13m 12.2M - 1.7.0 72c9c2088986 301adb4e9e22 2024-09-21T12:33:01.293 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi148 smithi148 *:9100 running (4m) 103s ago 13m 11.3M - 1.7.0 72c9c2088986 0cceff4bf342 2024-09-21T12:33:01.293 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (71s) 11s ago 12m 236M 1936M 19.3.0-5104-g44178494 b415816db8b4 7374add97534 2024-09-21T12:33:01.293 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (21s) 11s ago 12m 12.2M 1936M 19.3.0-5104-g44178494 b415816db8b4 14655a8da4e5 2024-09-21T12:33:01.293 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (11m) 11s ago 11m 466M 1936M 17.2.0 e1d6a67b021e 829fb8c2d827 2024-09-21T12:33:01.293 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (11m) 11s ago 11m 549M 1936M 17.2.0 e1d6a67b021e ef942c8556d0 2024-09-21T12:33:01.293 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi148 running (10m) 103s ago 10m 493M 2704M 17.2.0 e1d6a67b021e 62eefdd572c0 2024-09-21T12:33:01.294 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi148 running (10m) 103s ago 10m 314M 2704M 17.2.0 e1d6a67b021e 7b44be78baca 2024-09-21T12:33:01.294 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi148 running (10m) 103s ago 10m 509M 2704M 17.2.0 e1d6a67b021e 3d23057910ea 2024-09-21T12:33:01.294 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi148 running (9m) 103s ago 9m 334M 2704M 17.2.0 e1d6a67b021e f9e4db9d35fa 2024-09-21T12:33:01.294 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (3m) 11s ago 13m 49.4M - 2.51.0 1d3b7f56885b face2d4cdd1b 2024-09-21T12:33:01.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:01 smithi078 ceph-mon[63866]: from='client.34252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:33:01.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:01 smithi148 ceph-mon[49387]: from='client.34252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:33:01.822 INFO:teuthology.orchestra.run.smithi078.stdout: 5 182272 9.77 MB/sec execute 339 sec latency 480.868 ms 2024-09-21T12:33:01.961 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:33:01.961 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-21T12:33:01.961 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:33:01.961 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:33:01.961 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-21T12:33:01.961 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:33:01.961 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:33:01.961 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-21T12:33:01.961 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-21T12:33:01.962 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:33:01.962 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:33:01.962 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-21T12:33:01.962 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:33:01.962 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:33:01.962 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-21T12:33:01.962 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-09-21T12:33:01.962 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 6 2024-09-21T12:33:01.962 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T12:33:01.962 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:33:02.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:02 smithi078 ceph-mon[63866]: from='client.34256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:33:02.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:02 smithi078 ceph-mon[63866]: from='client.44183 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:33:02.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:02 smithi078 ceph-mon[63866]: osdmap e99: 8 total, 8 up, 8 in 2024-09-21T12:33:02.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:02 smithi078 ceph-mon[63866]: pgmap v121: 97 pgs: 2 active+undersized+remapped, 1 peering, 94 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.6 MiB/s wr, 438 op/s; 2.0 MiB/s, 57 objects/s recovering 2024-09-21T12:33:02.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:02 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 228/1617 objects degraded (14.100%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T12:33:02.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:02 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/4160296780' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:33:02.524 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:33:02.524 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5", 2024-09-21T12:33:02.525 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-21T12:33:02.525 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T12:33:02.525 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-21T12:33:02.525 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-21T12:33:02.525 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-09-21T12:33:02.525 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr" 2024-09-21T12:33:02.525 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-21T12:33:02.525 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "8/23 daemons upgraded", 2024-09-21T12:33:02.525 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T12:33:02.525 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-21T12:33:02.525 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:33:02.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:02 smithi148 ceph-mon[49387]: from='client.34256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:33:02.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:02 smithi148 ceph-mon[49387]: from='client.44183 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:33:02.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:02 smithi148 ceph-mon[49387]: osdmap e99: 8 total, 8 up, 8 in 2024-09-21T12:33:02.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:02 smithi148 ceph-mon[49387]: pgmap v121: 97 pgs: 2 active+undersized+remapped, 1 peering, 94 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.6 MiB/s wr, 438 op/s; 2.0 MiB/s, 57 objects/s recovering 2024-09-21T12:33:02.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:02 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 228/1617 objects degraded (14.100%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T12:33:02.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:02 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/4160296780' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:33:02.822 INFO:teuthology.orchestra.run.smithi078.stdout: 5 182587 9.75 MB/sec execute 340 sec latency 993.118 ms 2024-09-21T12:33:03.224 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-21T12:33:03.391 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:03 smithi078 ceph-mon[63866]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 228/1617 objects degraded (14.100%), 9 pgs degraded) 2024-09-21T12:33:03.391 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:03 smithi078 ceph-mon[63866]: Cluster is now healthy 2024-09-21T12:33:03.391 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:03 smithi078 ceph-mon[63866]: osdmap e100: 8 total, 8 up, 8 in 2024-09-21T12:33:03.391 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:03 smithi078 ceph-mon[63866]: from='client.34268 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:33:03.391 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:03 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/2623927049' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:33:03.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:03 smithi148 ceph-mon[49387]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 228/1617 objects degraded (14.100%), 9 pgs degraded) 2024-09-21T12:33:03.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:03 smithi148 ceph-mon[49387]: Cluster is now healthy 2024-09-21T12:33:03.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:03 smithi148 ceph-mon[49387]: osdmap e100: 8 total, 8 up, 8 in 2024-09-21T12:33:03.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:03 smithi148 ceph-mon[49387]: from='client.34268 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:33:03.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:03 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/2623927049' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:33:03.822 INFO:teuthology.orchestra.run.smithi078.stdout: 5 182836 9.74 MB/sec execute 341 sec latency 51.366 ms 2024-09-21T12:33:04.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:04 smithi078 ceph-mon[63866]: pgmap v123: 97 pgs: 2 active+undersized+remapped, 95 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.7 MiB/s wr, 434 op/s; 1.9 MiB/s, 54 objects/s recovering 2024-09-21T12:33:04.701 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:04 smithi148 ceph-mon[49387]: pgmap v123: 97 pgs: 2 active+undersized+remapped, 95 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.7 MiB/s wr, 434 op/s; 1.9 MiB/s, 54 objects/s recovering 2024-09-21T12:33:04.822 INFO:teuthology.orchestra.run.smithi078.stdout: 5 183170 9.75 MB/sec execute 342 sec latency 64.059 ms 2024-09-21T12:33:05.822 INFO:teuthology.orchestra.run.smithi078.stdout: 5 183590 9.74 MB/sec execute 343 sec latency 86.243 ms 2024-09-21T12:33:06.697 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:06 smithi078 ceph-mon[63866]: pgmap v124: 97 pgs: 2 active+undersized+remapped, 95 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.3 MiB/s wr, 306 op/s; 1.3 MiB/s, 38 objects/s recovering 2024-09-21T12:33:06.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 183923 9.73 MB/sec execute 344 sec latency 64.058 ms 2024-09-21T12:33:07.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:06 smithi148 ceph-mon[49387]: pgmap v124: 97 pgs: 2 active+undersized+remapped, 95 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.3 MiB/s wr, 306 op/s; 1.3 MiB/s, 38 objects/s recovering 2024-09-21T12:33:07.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 184184 9.73 MB/sec execute 345 sec latency 56.194 ms 2024-09-21T12:33:08.732 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:08 smithi078 ceph-mon[63866]: pgmap v125: 97 pgs: 97 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.0 MiB/s wr, 607 op/s; 1.1 MiB/s, 31 objects/s recovering 2024-09-21T12:33:08.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 184559 9.74 MB/sec execute 346 sec latency 48.421 ms 2024-09-21T12:33:09.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:08 smithi148 ceph-mon[49387]: pgmap v125: 97 pgs: 97 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.0 MiB/s wr, 607 op/s; 1.1 MiB/s, 31 objects/s recovering 2024-09-21T12:33:09.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 184889 9.74 MB/sec execute 347 sec latency 62.662 ms 2024-09-21T12:33:10.783 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:10 smithi078 ceph-mon[63866]: pgmap v126: 97 pgs: 97 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.1 MiB/s wr, 321 op/s 2024-09-21T12:33:10.822 INFO:teuthology.orchestra.run.smithi078.stdout: 5 185184 9.74 MB/sec execute 348 sec latency 86.096 ms 2024-09-21T12:33:11.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:10 smithi148 ceph-mon[49387]: pgmap v126: 97 pgs: 97 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.1 MiB/s wr, 321 op/s 2024-09-21T12:33:11.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 185383 9.74 MB/sec execute 349 sec latency 99.112 ms 2024-09-21T12:33:11.824 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:11 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T12:33:12.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:11 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T12:33:12.591 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:12 smithi078 ceph-mon[63866]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T12:33:12.591 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:12 smithi078 ceph-mon[63866]: Upgrade: osd.2 is safe to restart 2024-09-21T12:33:12.591 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:12 smithi078 ceph-mon[63866]: pgmap v127: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.7 MiB/s wr, 472 op/s 2024-09-21T12:33:12.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 185609 9.72 MB/sec execute 350 sec latency 67.176 ms 2024-09-21T12:33:13.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:12 smithi148 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T12:33:13.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:12 smithi148 ceph-mon[49387]: Upgrade: osd.2 is safe to restart 2024-09-21T12:33:13.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:12 smithi148 ceph-mon[49387]: pgmap v127: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.7 MiB/s wr, 472 op/s 2024-09-21T12:33:13.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 185934 9.72 MB/sec execute 351 sec latency 57.622 ms 2024-09-21T12:33:14.639 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:14 smithi078 ceph-mon[63866]: Upgrade: Updating osd.2 2024-09-21T12:33:14.639 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:14 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:14.639 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:14 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T12:33:14.639 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:14 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:33:14.639 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:14 smithi078 ceph-mon[63866]: Deploying daemon osd.2 on smithi078 2024-09-21T12:33:14.640 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:14 smithi078 ceph-mon[63866]: pgmap v128: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 428 op/s 2024-09-21T12:33:14.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:14 smithi148 ceph-mon[49387]: Upgrade: Updating osd.2 2024-09-21T12:33:14.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:14 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:14.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:14 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T12:33:14.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:14 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:33:14.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:14 smithi148 ceph-mon[49387]: Deploying daemon osd.2 on smithi078 2024-09-21T12:33:14.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:14 smithi148 ceph-mon[49387]: pgmap v128: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 428 op/s 2024-09-21T12:33:14.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 186268 9.70 MB/sec execute 352 sec latency 34.806 ms 2024-09-21T12:33:15.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 186585 9.71 MB/sec execute 353 sec latency 36.938 ms 2024-09-21T12:33:15.923 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:15 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:15.923 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:15 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:33:15.923 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:15 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:15.923 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:15 smithi078 ceph-mon[63866]: pgmap v129: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.0 MiB/s wr, 399 op/s 2024-09-21T12:33:16.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:15 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:16.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:15 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:33:16.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:15 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:16.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:15 smithi148 ceph-mon[49387]: pgmap v129: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.0 MiB/s wr, 399 op/s 2024-09-21T12:33:16.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 186916 9.70 MB/sec execute 354 sec latency 86.391 ms 2024-09-21T12:33:17.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 187248 9.68 MB/sec execute 355 sec latency 53.095 ms 2024-09-21T12:33:18.245 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:18 smithi078 systemd[1]: Stopping Ceph osd.2 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:33:18.508 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:18 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2[41869]: 2024-09-21T12:33:18.398+0000 7fe573aac700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T12:33:18.508 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:18 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2[41869]: 2024-09-21T12:33:18.398+0000 7fe573aac700 -1 osd.2 100 *** Got signal Terminated *** 2024-09-21T12:33:18.508 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:18 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2[41869]: 2024-09-21T12:33:18.398+0000 7fe573aac700 -1 osd.2 100 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T12:33:18.789 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:18 smithi078 ceph-mon[63866]: pgmap v130: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.9 MiB/s wr, 589 op/s 2024-09-21T12:33:18.789 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:18 smithi078 ceph-mon[63866]: osd.2 marked itself down and dead 2024-09-21T12:33:18.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 187482 9.67 MB/sec execute 356 sec latency 257.535 ms 2024-09-21T12:33:19.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:18 smithi148 ceph-mon[49387]: pgmap v130: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.9 MiB/s wr, 589 op/s 2024-09-21T12:33:19.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:18 smithi148 ceph-mon[49387]: osd.2 marked itself down and dead 2024-09-21T12:33:19.556 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:19 smithi078 podman[73181]: 2024-09-21 12:33:19.211798622 +0000 UTC m=+1.003277968 container died 829fb8c2d82760b8e417b82de269af9aef07ab764f9179db8f1066308cae4d2e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2, ceph=True, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, release=754, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, version=8, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, io.k8s.display-name=CentOS Stream 8) 2024-09-21T12:33:19.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 187519 9.65 MB/sec execute 357 sec latency 1028.743 ms 2024-09-21T12:33:19.834 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:19 smithi078 ceph-mon[63866]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T12:33:19.834 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:19 smithi078 ceph-mon[63866]: osdmap e101: 8 total, 7 up, 8 in 2024-09-21T12:33:19.834 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:19 smithi078 ceph-mon[63866]: osdmap e102: 8 total, 7 up, 8 in 2024-09-21T12:33:20.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:19 smithi148 ceph-mon[49387]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T12:33:20.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:19 smithi148 ceph-mon[49387]: osdmap e101: 8 total, 7 up, 8 in 2024-09-21T12:33:20.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:19 smithi148 ceph-mon[49387]: osdmap e102: 8 total, 7 up, 8 in 2024-09-21T12:33:20.089 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:20 smithi078 podman[73181]: 2024-09-21 12:33:20.033494009 +0000 UTC m=+1.824973349 container cleanup 829fb8c2d82760b8e417b82de269af9aef07ab764f9179db8f1066308cae4d2e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, version=8, release=754, io.openshift.expose-services=, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., RELEASE=HEAD, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, distribution-scope=public, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-21T12:33:20.089 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:20 smithi078 bash[73181]: ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2 2024-09-21T12:33:20.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:20 smithi078 ceph-mon[63866]: pgmap v132: 97 pgs: 5 stale+active+clean, 92 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 451 op/s 2024-09-21T12:33:20.606 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:20 smithi078 podman[73197]: 2024-09-21 12:33:20.362227138 +0000 UTC m=+1.153576069 container remove 829fb8c2d82760b8e417b82de269af9aef07ab764f9179db8f1066308cae4d2e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, ceph=True, com.redhat.component=centos-stream-container, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , distribution-scope=public, architecture=x86_64, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream) 2024-09-21T12:33:20.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 187763 9.65 MB/sec execute 358 sec latency 60.818 ms 2024-09-21T12:33:21.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:20 smithi148 ceph-mon[49387]: pgmap v132: 97 pgs: 5 stale+active+clean, 92 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 451 op/s 2024-09-21T12:33:21.122 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:20 smithi078 podman[73279]: 2024-09-21 12:33:20.837038111 +0000 UTC m=+0.018749332 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:33:21.122 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:21 smithi078 podman[73279]: 2024-09-21 12:33:21.014039262 +0000 UTC m=+0.195750476 container create 262fe524baea1b8e06ae88e2f61547de410bb48182ee5ec80382f71200b74952 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-guits-main-2024-09-21-0842-4417849, CEPH_POINT_RELEASE=) 2024-09-21T12:33:21.385 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:21 smithi078 podman[73279]: 2024-09-21 12:33:21.359542764 +0000 UTC m=+0.541253994 container init 262fe524baea1b8e06ae88e2f61547de410bb48182ee5ec80382f71200b74952 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-21T12:33:21.386 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:21 smithi078 podman[73279]: 2024-09-21 12:33:21.364463179 +0000 UTC m=+0.546174393 container start 262fe524baea1b8e06ae88e2f61547de410bb48182ee5ec80382f71200b74952 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T12:33:21.643 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:21 smithi078 podman[73279]: 2024-09-21 12:33:21.553126305 +0000 UTC m=+0.734837521 container attach 262fe524baea1b8e06ae88e2f61547de410bb48182ee5ec80382f71200b74952 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0842-4417849, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True) 2024-09-21T12:33:21.643 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:21 smithi078 ceph-mon[63866]: Health check failed: Degraded data redundancy: 139/1527 objects degraded (9.103%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T12:33:21.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 188024 9.65 MB/sec execute 359 sec latency 72.981 ms 2024-09-21T12:33:21.912 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:21 smithi078 podman[73279]: 2024-09-21 12:33:21.668198877 +0000 UTC m=+0.849910091 container died 262fe524baea1b8e06ae88e2f61547de410bb48182ee5ec80382f71200b74952 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2-deactivate, ceph=True, RELEASE=wip-guits-main-2024-09-21-0842-4417849, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-21T12:33:22.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:21 smithi148 ceph-mon[49387]: Health check failed: Degraded data redundancy: 139/1527 objects degraded (9.103%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T12:33:22.676 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:22 smithi078 ceph-mon[63866]: pgmap v134: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.7 MiB/s wr, 504 op/s; 139/1527 objects degraded (9.103%) 2024-09-21T12:33:22.676 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:22 smithi078 podman[73301]: 2024-09-21 12:33:22.65253568 +0000 UTC m=+0.973640113 container remove 262fe524baea1b8e06ae88e2f61547de410bb48182ee5ec80382f71200b74952 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-21T12:33:22.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 188418 9.65 MB/sec execute 360 sec latency 49.482 ms 2024-09-21T12:33:22.927 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:22 smithi078 systemd[1]: ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.2.service: Deactivated successfully. 2024-09-21T12:33:22.927 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:22 smithi078 systemd[1]: Stopped Ceph osd.2 for 6263deca-7813-11ef-baf6-efdc52797490. 2024-09-21T12:33:22.927 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:22 smithi078 systemd[1]: ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.2.service: Consumed 2min 27.939s CPU time. 2024-09-21T12:33:23.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:22 smithi148 ceph-mon[49387]: pgmap v134: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.7 MiB/s wr, 504 op/s; 139/1527 objects degraded (9.103%) 2024-09-21T12:33:23.178 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:23 smithi078 systemd[1]: Starting Ceph osd.2 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:33:23.435 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:23 smithi078 podman[73393]: 2024-09-21 12:33:23.233579226 +0000 UTC m=+0.023513060 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:33:23.435 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:23 smithi078 podman[73393]: 2024-09-21 12:33:23.36217508 +0000 UTC m=+0.152108920 container create c4af6ba0e7a0c1813a82f62b75e1e39c1e403687d1ae7111eba7d337fb39ee6c (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2-activate, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-21T12:33:23.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 188761 9.66 MB/sec execute 361 sec latency 44.830 ms 2024-09-21T12:33:23.949 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:23 smithi078 podman[73393]: 2024-09-21 12:33:23.724088888 +0000 UTC m=+0.514022730 container init c4af6ba0e7a0c1813a82f62b75e1e39c1e403687d1ae7111eba7d337fb39ee6c (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2-activate, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-21T12:33:23.949 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:23 smithi078 podman[73393]: 2024-09-21 12:33:23.736712201 +0000 UTC m=+0.526646040 container start c4af6ba0e7a0c1813a82f62b75e1e39c1e403687d1ae7111eba7d337fb39ee6c (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2-activate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-21T12:33:23.949 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:23 smithi078 podman[73393]: 2024-09-21 12:33:23.896513092 +0000 UTC m=+0.686446933 container attach c4af6ba0e7a0c1813a82f62b75e1e39c1e403687d1ae7111eba7d337fb39ee6c (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2-activate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-21T12:33:23.949 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:23 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2-activate[73407]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:33:23.949 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:23 smithi078 bash[73393]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:33:24.200 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:23 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2-activate[73407]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:33:24.201 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:23 smithi078 bash[73393]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:33:24.703 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:24 smithi078 ceph-mon[63866]: pgmap v135: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.7 MiB/s wr, 504 op/s; 139/1527 objects degraded (9.103%) 2024-09-21T12:33:24.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 189011 9.66 MB/sec execute 362 sec latency 40.151 ms 2024-09-21T12:33:25.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:24 smithi148 ceph-mon[49387]: pgmap v135: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.7 MiB/s wr, 504 op/s; 139/1527 objects degraded (9.103%) 2024-09-21T12:33:25.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 189514 9.66 MB/sec execute 363 sec latency 43.842 ms 2024-09-21T12:33:26.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:26 smithi078 ceph-mon[63866]: pgmap v136: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.8 MiB/s wr, 219 op/s; 139/1527 objects degraded (9.103%) 2024-09-21T12:33:26.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 189959 9.65 MB/sec execute 364 sec latency 42.953 ms 2024-09-21T12:33:27.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:26 smithi148 ceph-mon[49387]: pgmap v136: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.8 MiB/s wr, 219 op/s; 139/1527 objects degraded (9.103%) 2024-09-21T12:33:27.725 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:27 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 148/1683 objects degraded (8.794%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T12:33:27.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 190282 9.65 MB/sec execute 365 sec latency 69.543 ms 2024-09-21T12:33:28.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:27 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 148/1683 objects degraded (8.794%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T12:33:28.505 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:28 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2-activate[73407]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T12:33:28.505 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:28 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2-activate[73407]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:33:28.505 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:28 smithi078 bash[73393]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T12:33:28.505 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:28 smithi078 bash[73393]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:33:28.505 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:28 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2-activate[73407]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:33:28.505 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:28 smithi078 bash[73393]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:33:28.505 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:28 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2-activate[73407]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T12:33:28.505 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:28 smithi078 bash[73393]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T12:33:28.505 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:28 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2-activate[73407]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-09-21T12:33:28.505 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:28 smithi078 bash[73393]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-09-21T12:33:28.765 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:28 smithi078 ceph-mon[63866]: pgmap v137: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 350 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.9 MiB/s wr, 474 op/s; 148/1683 objects degraded (8.794%) 2024-09-21T12:33:28.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 190675 9.65 MB/sec execute 366 sec latency 77.111 ms 2024-09-21T12:33:29.017 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:28 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2-activate[73407]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-21T12:33:29.017 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:28 smithi078 bash[73393]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-21T12:33:29.017 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:28 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2-activate[73407]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-09-21T12:33:29.017 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:28 smithi078 bash[73393]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-09-21T12:33:29.018 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:28 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2-activate[73407]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T12:33:29.018 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:28 smithi078 bash[73393]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T12:33:29.018 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:28 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2-activate[73407]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T12:33:29.018 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:28 smithi078 bash[73393]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T12:33:29.018 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:28 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2-activate[73407]: --> ceph-volume lvm activate successful for osd ID: 2 2024-09-21T12:33:29.018 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:28 smithi078 bash[73393]: --> ceph-volume lvm activate successful for osd ID: 2 2024-09-21T12:33:29.018 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:28 smithi078 podman[73393]: 2024-09-21 12:33:28.88490162 +0000 UTC m=+5.674835458 container died c4af6ba0e7a0c1813a82f62b75e1e39c1e403687d1ae7111eba7d337fb39ee6c (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2-activate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-21T12:33:29.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:28 smithi148 ceph-mon[49387]: pgmap v137: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 350 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.9 MiB/s wr, 474 op/s; 148/1683 objects degraded (8.794%) 2024-09-21T12:33:29.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 191013 9.64 MB/sec execute 367 sec latency 80.276 ms 2024-09-21T12:33:30.291 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:30 smithi078 podman[73643]: 2024-09-21 12:33:30.152951914 +0000 UTC m=+1.258896183 container remove c4af6ba0e7a0c1813a82f62b75e1e39c1e403687d1ae7111eba7d337fb39ee6c (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2-activate, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-21-0842-4417849) 2024-09-21T12:33:30.556 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:30 smithi078 podman[73701]: 2024-09-21 12:33:30.284795083 +0000 UTC m=+0.018074872 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:33:30.556 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:30 smithi078 podman[73701]: 2024-09-21 12:33:30.470859799 +0000 UTC m=+0.204139578 container create 07ceda32f383726003b3f2389e763a2ec80e9d0cbbc4edf9f4a361f4f2fc5c2a (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-21T12:33:30.809 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:30 smithi078 ceph-mon[63866]: pgmap v138: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 350 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.4 MiB/s wr, 424 op/s; 148/1683 objects degraded (8.794%) 2024-09-21T12:33:30.810 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:30.810 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:33:30.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 191284 9.64 MB/sec execute 368 sec latency 59.592 ms 2024-09-21T12:33:31.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:30 smithi148 ceph-mon[49387]: pgmap v138: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 350 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.4 MiB/s wr, 424 op/s; 148/1683 objects degraded (8.794%) 2024-09-21T12:33:31.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:31.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:33:31.079 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:30 smithi078 podman[73701]: 2024-09-21 12:33:30.864680206 +0000 UTC m=+0.597960003 container init 07ceda32f383726003b3f2389e763a2ec80e9d0cbbc4edf9f4a361f4f2fc5c2a (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-21T12:33:31.079 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:30 smithi078 podman[73701]: 2024-09-21 12:33:30.873906807 +0000 UTC m=+0.607186587 container start 07ceda32f383726003b3f2389e763a2ec80e9d0cbbc4edf9f4a361f4f2fc5c2a (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T12:33:31.079 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:30 smithi078 ceph-osd[73720]: -- 172.21.15.78:0/3320596896 <== mon.0 v2:172.21.15.78:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x562e140881e0 con 0x562e14014000 2024-09-21T12:33:31.079 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:31 smithi078 bash[73701]: 07ceda32f383726003b3f2389e763a2ec80e9d0cbbc4edf9f4a361f4f2fc5c2a 2024-09-21T12:33:31.079 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:31 smithi078 systemd[1]: Started Ceph osd.2 for 6263deca-7813-11ef-baf6-efdc52797490. 2024-09-21T12:33:31.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 191797 9.66 MB/sec execute 369 sec latency 43.781 ms 2024-09-21T12:33:32.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:32 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:32.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:32 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:32.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:32 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:33:32.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:32 smithi078 ceph-mon[63866]: pgmap v139: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.3 MiB/s wr, 567 op/s; 138/1602 objects degraded (8.614%) 2024-09-21T12:33:32.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:32 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:32.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:32 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:32.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:32 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:33:32.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:32 smithi148 ceph-mon[49387]: pgmap v139: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.3 MiB/s wr, 567 op/s; 138/1602 objects degraded (8.614%) 2024-09-21T12:33:32.603 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:32 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2[73716]: 2024-09-21T12:33:32.545+0000 7f43922ef640 -1 Falling back to public interface 2024-09-21T12:33:32.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 192292 9.68 MB/sec execute 370 sec latency 43.479 ms 2024-09-21T12:33:33.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:33 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 138/1614 objects degraded (8.550%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T12:33:33.810 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-21T12:33:33.823 INFO:teuthology.orchestra.run.smithi078.stdout: 5 192684 9.69 MB/sec execute 371 sec latency 35.447 ms 2024-09-21T12:33:34.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:33 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 138/1614 objects degraded (8.550%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T12:33:34.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:34 smithi078 ceph-mon[63866]: pgmap v140: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.5 MiB/s wr, 435 op/s; 138/1614 objects degraded (8.550%) 2024-09-21T12:33:34.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:34 smithi078 ceph-mon[63866]: from='client.34278 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:33:34.815 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T12:33:34.815 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (5m) 45s ago 14m 19.3M - 0.27.0 11f11916f8cd e6e67ab07dec 2024-09-21T12:33:34.815 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (2m) 45s ago 14m 6895k - 19.3.0-5104-g44178494 b415816db8b4 46a10ea0b111 2024-09-21T12:33:34.815 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi148 smithi148 running (2m) 2m ago 14m 6895k - 19.3.0-5104-g44178494 b415816db8b4 23deb0615621 2024-09-21T12:33:34.815 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (5m) 45s ago 14m 53.4M - dad864ee21e9 2a140c36091b 2024-09-21T12:33:34.816 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.ytpzwh smithi078 running (8m) 45s ago 8m 37.8M - 17.2.0 e1d6a67b021e 5e9b109502a2 2024-09-21T12:33:34.816 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi148.uukezr smithi148 running (8m) 2m ago 8m 13.7M - 17.2.0 e1d6a67b021e b7e8aa0f3a6c 2024-09-21T12:33:34.816 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.ntwyan smithi078 *:8443,9283,8765 running (4m) 45s ago 16m 516M - 19.3.0-5104-g44178494 b415816db8b4 a61baedebf43 2024-09-21T12:33:34.816 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi148.oyiqmx smithi148 *:8443,9283,8765 running (4m) 2m ago 14m 454M - 19.3.0-5104-g44178494 b415816db8b4 0255535c6fdb 2024-09-21T12:33:34.816 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (3m) 45s ago 16m 53.2M 2048M 19.3.0-5104-g44178494 b415816db8b4 22b3097efbd8 2024-09-21T12:33:34.816 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi148 smithi148 running (3m) 2m ago 14m 36.1M 2048M 19.3.0-5104-g44178494 b415816db8b4 df8b08509136 2024-09-21T12:33:34.816 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.ydbhru smithi078 *:2049 running (8m) 45s ago 8m 278M - 4.0 e1d6a67b021e 7e5b0274ac38 2024-09-21T12:33:34.816 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi148.einuzt smithi148 *:2049 running (8m) 2m ago 8m 86.8M - 4.0 e1d6a67b021e 76a8b0eabede 2024-09-21T12:33:34.816 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (5m) 45s ago 14m 12.2M - 1.7.0 72c9c2088986 301adb4e9e22 2024-09-21T12:33:34.816 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi148 smithi148 *:9100 running (5m) 2m ago 13m 11.3M - 1.7.0 72c9c2088986 0cceff4bf342 2024-09-21T12:33:34.816 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (105s) 45s ago 13m 236M 1936M 19.3.0-5104-g44178494 b415816db8b4 7374add97534 2024-09-21T12:33:34.816 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (55s) 45s ago 12m 12.2M 1936M 19.3.0-5104-g44178494 b415816db8b4 14655a8da4e5 2024-09-21T12:33:34.816 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 starting - - - 1936M 2024-09-21T12:33:34.816 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (11m) 45s ago 11m 549M 1936M 17.2.0 e1d6a67b021e ef942c8556d0 2024-09-21T12:33:34.816 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi148 running (11m) 2m ago 11m 493M 2704M 17.2.0 e1d6a67b021e 62eefdd572c0 2024-09-21T12:33:34.817 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi148 running (11m) 2m ago 11m 314M 2704M 17.2.0 e1d6a67b021e 7b44be78baca 2024-09-21T12:33:34.817 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi148 running (10m) 2m ago 10m 509M 2704M 17.2.0 e1d6a67b021e 3d23057910ea 2024-09-21T12:33:34.817 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi148 running (10m) 2m ago 10m 334M 2704M 17.2.0 e1d6a67b021e f9e4db9d35fa 2024-09-21T12:33:34.817 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (4m) 45s ago 13m 49.4M - 2.51.0 1d3b7f56885b face2d4cdd1b 2024-09-21T12:33:34.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 193004 9.68 MB/sec execute 372 sec latency 49.970 ms 2024-09-21T12:33:35.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:34 smithi148 ceph-mon[49387]: pgmap v140: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.5 MiB/s wr, 435 op/s; 138/1614 objects degraded (8.550%) 2024-09-21T12:33:35.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:34 smithi148 ceph-mon[49387]: from='client.34278 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:33:35.484 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:33:35.484 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-21T12:33:35.484 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:33:35.484 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:33:35.484 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-21T12:33:35.484 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:33:35.485 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:33:35.485 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-21T12:33:35.485 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-21T12:33:35.485 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:33:35.485 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:33:35.485 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-21T12:33:35.485 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:33:35.485 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:33:35.485 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-21T12:33:35.485 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-21T12:33:35.485 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 6 2024-09-21T12:33:35.485 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T12:33:35.485 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:33:35.666 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:35 smithi078 ceph-mon[63866]: from='client.44205 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:33:35.666 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:35 smithi078 ceph-mon[63866]: from='client.44209 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:33:35.666 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:35 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/2478762871' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:33:35.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 193398 9.67 MB/sec execute 373 sec latency 44.008 ms 2024-09-21T12:33:35.975 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:33:35.975 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5", 2024-09-21T12:33:35.975 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-21T12:33:35.975 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T12:33:35.975 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-21T12:33:35.975 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-21T12:33:35.975 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-09-21T12:33:35.976 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr" 2024-09-21T12:33:35.976 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-21T12:33:35.976 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "8/23 daemons upgraded", 2024-09-21T12:33:35.976 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T12:33:35.976 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-21T12:33:35.976 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:33:36.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:35 smithi148 ceph-mon[49387]: from='client.44205 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:33:36.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:35 smithi148 ceph-mon[49387]: from='client.44209 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:33:36.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:35 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/2478762871' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:33:36.665 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 141/1626 objects degraded (8.672%), 19 pgs degraded 2024-09-21T12:33:36.665 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-21T12:33:36.665 INFO:teuthology.orchestra.run.smithi078.stdout: osd.2 (root=default,host=smithi078) is down 2024-09-21T12:33:36.665 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 141/1626 objects degraded (8.672%), 19 pgs degraded 2024-09-21T12:33:36.665 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1 is active+undersized+degraded, acting [3,0] 2024-09-21T12:33:36.665 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.3 is active+undersized+degraded, acting [5,7] 2024-09-21T12:33:36.666 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.7 is active+undersized+degraded, acting [6,7] 2024-09-21T12:33:36.666 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.c is active+undersized+degraded, acting [5,0] 2024-09-21T12:33:36.666 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.e is active+undersized+degraded, acting [3,7] 2024-09-21T12:33:36.666 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.10 is active+undersized+degraded, acting [1,0] 2024-09-21T12:33:36.666 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.16 is active+undersized+degraded, acting [5,6] 2024-09-21T12:33:36.666 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.17 is active+undersized+degraded, acting [6,5] 2024-09-21T12:33:36.666 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.18 is active+undersized+degraded, acting [5,4] 2024-09-21T12:33:36.666 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1c is active+undersized+degraded, acting [7,5] 2024-09-21T12:33:36.666 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.0 is active+undersized+degraded, acting [1,6] 2024-09-21T12:33:36.666 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.4 is active+undersized+degraded, acting [1,5] 2024-09-21T12:33:36.666 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.5 is active+undersized+degraded, acting [5,3] 2024-09-21T12:33:36.666 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.9 is active+undersized+degraded, acting [4,7] 2024-09-21T12:33:36.666 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.13 is active+undersized+degraded, acting [7,4] 2024-09-21T12:33:36.666 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1a is active+undersized+degraded, acting [4,1] 2024-09-21T12:33:36.667 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1e is active+undersized+degraded, acting [3,6] 2024-09-21T12:33:36.667 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1f is active+undersized+degraded, acting [0,5] 2024-09-21T12:33:36.667 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.19 is active+undersized+degraded, acting [3,0] 2024-09-21T12:33:36.689 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:36 smithi078 ceph-mon[63866]: pgmap v141: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.3 MiB/s wr, 555 op/s; 141/1626 objects degraded (8.672%) 2024-09-21T12:33:36.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 193703 9.67 MB/sec execute 374 sec latency 86.908 ms 2024-09-21T12:33:37.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:36 smithi148 ceph-mon[49387]: pgmap v141: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.3 MiB/s wr, 555 op/s; 141/1626 objects degraded (8.672%) 2024-09-21T12:33:37.699 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:37 smithi078 ceph-mon[63866]: from='client.34290 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:33:37.700 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:37 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/1807149839' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:33:37.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 194138 9.67 MB/sec execute 375 sec latency 38.108 ms 2024-09-21T12:33:38.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:37 smithi148 ceph-mon[49387]: from='client.34290 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:33:38.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:37 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/1807149839' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:33:38.709 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:38 smithi078 ceph-mon[63866]: pgmap v142: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 7.2 MiB/s wr, 707 op/s; 153/1728 objects degraded (8.854%) 2024-09-21T12:33:38.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 194601 9.67 MB/sec execute 376 sec latency 50.389 ms 2024-09-21T12:33:39.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:38 smithi148 ceph-mon[49387]: pgmap v142: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 7.2 MiB/s wr, 707 op/s; 153/1728 objects degraded (8.854%) 2024-09-21T12:33:39.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 194983 9.68 MB/sec execute 377 sec latency 36.822 ms 2024-09-21T12:33:40.759 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:40 smithi078 ceph-mon[63866]: pgmap v143: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 499 op/s; 153/1728 objects degraded (8.854%) 2024-09-21T12:33:40.759 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:40 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:40.759 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:40 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:40.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 195392 9.69 MB/sec execute 378 sec latency 49.785 ms 2024-09-21T12:33:41.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:40 smithi148 ceph-mon[49387]: pgmap v143: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 499 op/s; 153/1728 objects degraded (8.854%) 2024-09-21T12:33:41.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:40 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:41.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:40 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:41.265 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:41 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2[73716]: 2024-09-21T12:33:41.115+0000 7f43922ef640 -1 osd.2 0 read_superblock omap replica is missing. 2024-09-21T12:33:41.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 195797 9.70 MB/sec execute 379 sec latency 51.013 ms 2024-09-21T12:33:42.019 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:42 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:42.019 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:42 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:42.019 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:42 smithi078 ceph-mon[63866]: pgmap v144: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 714 op/s; 144/1650 objects degraded (8.727%) 2024-09-21T12:33:42.019 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:42 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 153/1728 objects degraded (8.854%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T12:33:42.270 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:42 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2[73716]: 2024-09-21T12:33:42.134+0000 7f43922ef640 -1 osd.2 100 log_to_monitors true 2024-09-21T12:33:42.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:42 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:42.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:42 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:42.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:42 smithi148 ceph-mon[49387]: pgmap v144: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 714 op/s; 144/1650 objects degraded (8.727%) 2024-09-21T12:33:42.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:42 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 153/1728 objects degraded (8.854%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T12:33:42.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 196128 9.71 MB/sec execute 380 sec latency 46.069 ms 2024-09-21T12:33:43.038 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:43 smithi078 ceph-mon[63866]: from='osd.2 [v2:172.21.15.78:6818/938589223,v1:172.21.15.78:6819/938589223]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T12:33:43.039 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:43 smithi078 ceph-mon[63866]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T12:33:43.290 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:33:43 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2[73716]: 2024-09-21T12:33:43.093+0000 7f438a099640 -1 osd.2 100 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T12:33:43.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:43 smithi148 ceph-mon[49387]: from='osd.2 [v2:172.21.15.78:6818/938589223,v1:172.21.15.78:6819/938589223]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T12:33:43.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:43 smithi148 ceph-mon[49387]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T12:33:43.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 196497 9.71 MB/sec execute 381 sec latency 35.914 ms 2024-09-21T12:33:44.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:44 smithi078 ceph-mon[63866]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T12:33:44.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:44 smithi078 ceph-mon[63866]: from='osd.2 [v2:172.21.15.78:6818/938589223,v1:172.21.15.78:6819/938589223]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]} : dispatch 2024-09-21T12:33:44.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:44 smithi078 ceph-mon[63866]: osdmap e103: 8 total, 7 up, 8 in 2024-09-21T12:33:44.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:44 smithi078 ceph-mon[63866]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]} : dispatch 2024-09-21T12:33:44.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:44 smithi078 ceph-mon[63866]: pgmap v146: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.9 MiB/s wr, 583 op/s; 144/1650 objects degraded (8.727%) 2024-09-21T12:33:44.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:44 smithi148 ceph-mon[49387]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T12:33:44.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:44 smithi148 ceph-mon[49387]: from='osd.2 [v2:172.21.15.78:6818/938589223,v1:172.21.15.78:6819/938589223]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]} : dispatch 2024-09-21T12:33:44.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:44 smithi148 ceph-mon[49387]: osdmap e103: 8 total, 7 up, 8 in 2024-09-21T12:33:44.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:44 smithi148 ceph-mon[49387]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi078", "root=default"]} : dispatch 2024-09-21T12:33:44.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:44 smithi148 ceph-mon[49387]: pgmap v146: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.9 MiB/s wr, 583 op/s; 144/1650 objects degraded (8.727%) 2024-09-21T12:33:44.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 196641 9.68 MB/sec execute 382 sec latency 762.756 ms 2024-09-21T12:33:45.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:45 smithi148 ceph-mon[49387]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T12:33:45.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:45 smithi148 ceph-mon[49387]: osd.2 [v2:172.21.15.78:6818/938589223,v1:172.21.15.78:6819/938589223] boot 2024-09-21T12:33:45.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:45 smithi148 ceph-mon[49387]: osdmap e104: 8 total, 8 up, 8 in 2024-09-21T12:33:45.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T12:33:45.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:45.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:33:45.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:45 smithi078 ceph-mon[63866]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T12:33:45.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:45 smithi078 ceph-mon[63866]: osd.2 [v2:172.21.15.78:6818/938589223,v1:172.21.15.78:6819/938589223] boot 2024-09-21T12:33:45.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:45 smithi078 ceph-mon[63866]: osdmap e104: 8 total, 8 up, 8 in 2024-09-21T12:33:45.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T12:33:45.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:45.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:33:45.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 196641 9.66 MB/sec execute 383 sec latency 1762.783 ms 2024-09-21T12:33:46.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:46 smithi148 ceph-mon[49387]: osdmap e105: 8 total, 8 up, 8 in 2024-09-21T12:33:46.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:46 smithi148 ceph-mon[49387]: pgmap v149: 97 pgs: 2 remapped+peering, 7 active+undersized, 17 active+undersized+degraded, 71 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.2 MiB/s wr, 678 op/s; 123/1617 objects degraded (7.607%) 2024-09-21T12:33:46.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:46 smithi148 ceph-mon[49387]: osdmap e106: 8 total, 8 up, 8 in 2024-09-21T12:33:46.305 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:46 smithi078 ceph-mon[63866]: osdmap e105: 8 total, 8 up, 8 in 2024-09-21T12:33:46.305 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:46 smithi078 ceph-mon[63866]: pgmap v149: 97 pgs: 2 remapped+peering, 7 active+undersized, 17 active+undersized+degraded, 71 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.2 MiB/s wr, 678 op/s; 123/1617 objects degraded (7.607%) 2024-09-21T12:33:46.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:46 smithi078 ceph-mon[63866]: osdmap e106: 8 total, 8 up, 8 in 2024-09-21T12:33:46.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 196820 9.64 MB/sec execute 384 sec latency 2022.940 ms 2024-09-21T12:33:47.078 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:47 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 123/1617 objects degraded (7.607%), 17 pgs degraded (PG_DEGRADED) 2024-09-21T12:33:47.078 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:47 smithi078 ceph-mon[63866]: osdmap e107: 8 total, 8 up, 8 in 2024-09-21T12:33:47.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:47 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 123/1617 objects degraded (7.607%), 17 pgs degraded (PG_DEGRADED) 2024-09-21T12:33:47.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:47 smithi148 ceph-mon[49387]: osdmap e107: 8 total, 8 up, 8 in 2024-09-21T12:33:47.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 197075 9.62 MB/sec execute 385 sec latency 747.333 ms 2024-09-21T12:33:48.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:48 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:48.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:48 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:48.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:48 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:33:48.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:48 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:33:48.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:48 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:48.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:48 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:48.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:48 smithi078 ceph-mon[63866]: pgmap v152: 97 pgs: 1 active+recovering+undersized+remapped, 2 active+undersized+remapped, 8 remapped+peering, 1 active+undersized, 2 active+undersized+degraded, 83 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.5 MiB/s wr, 494 op/s; 10/1641 objects degraded (0.609%); 2.0 MiB/s, 8 objects/s recovering 2024-09-21T12:33:48.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:48 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:33:48.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:48 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:33:48.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:48 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:33:48.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:48 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:33:48.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:48 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T12:33:48.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:48 smithi078 ceph-mon[63866]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T12:33:48.494 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:48 smithi078 ceph-mon[63866]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-09-21T12:33:48.494 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:48 smithi078 ceph-mon[63866]: osdmap e108: 8 total, 8 up, 8 in 2024-09-21T12:33:48.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:48 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:48.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:48 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:48.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:48 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:33:48.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:48 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:33:48.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:48 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:48.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:48 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:33:48.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:48 smithi148 ceph-mon[49387]: pgmap v152: 97 pgs: 1 active+recovering+undersized+remapped, 2 active+undersized+remapped, 8 remapped+peering, 1 active+undersized, 2 active+undersized+degraded, 83 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.5 MiB/s wr, 494 op/s; 10/1641 objects degraded (0.609%); 2.0 MiB/s, 8 objects/s recovering 2024-09-21T12:33:48.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:48 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:33:48.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:48 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:33:48.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:48 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:33:48.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:48 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:33:48.802 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:48 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T12:33:48.802 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:48 smithi148 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T12:33:48.802 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:48 smithi148 ceph-mon[49387]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-09-21T12:33:48.802 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:48 smithi148 ceph-mon[49387]: osdmap e108: 8 total, 8 up, 8 in 2024-09-21T12:33:48.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 197196 9.61 MB/sec execute 386 sec latency 887.310 ms 2024-09-21T12:33:49.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 197422 9.60 MB/sec execute 387 sec latency 897.552 ms 2024-09-21T12:33:50.014 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:49 smithi078 ceph-mon[63866]: osdmap e109: 8 total, 8 up, 8 in 2024-09-21T12:33:50.015 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:49 smithi078 ceph-mon[63866]: pgmap v155: 97 pgs: 1 active+recovering+undersized+remapped, 8 remapped+peering, 88 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 2.7 MiB/s wr, 121 op/s; 2.0 MiB/s, 8 objects/s recovering 2024-09-21T12:33:50.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:49 smithi148 ceph-mon[49387]: osdmap e109: 8 total, 8 up, 8 in 2024-09-21T12:33:50.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:49 smithi148 ceph-mon[49387]: pgmap v155: 97 pgs: 1 active+recovering+undersized+remapped, 8 remapped+peering, 88 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 2.7 MiB/s wr, 121 op/s; 2.0 MiB/s, 8 objects/s recovering 2024-09-21T12:33:50.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 197704 9.59 MB/sec execute 388 sec latency 92.119 ms 2024-09-21T12:33:51.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:50 smithi078 ceph-mon[63866]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/1641 objects degraded (0.609%), 2 pgs degraded) 2024-09-21T12:33:51.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:50 smithi078 ceph-mon[63866]: Cluster is now healthy 2024-09-21T12:33:51.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:50 smithi148 ceph-mon[49387]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/1641 objects degraded (0.609%), 2 pgs degraded) 2024-09-21T12:33:51.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:50 smithi148 ceph-mon[49387]: Cluster is now healthy 2024-09-21T12:33:51.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 198121 9.59 MB/sec execute 389 sec latency 73.313 ms 2024-09-21T12:33:52.078 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:51 smithi078 ceph-mon[63866]: pgmap v156: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 2.7 MiB/s wr, 321 op/s; 2.3 MiB/s, 41 objects/s recovering 2024-09-21T12:33:52.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:51 smithi148 ceph-mon[49387]: pgmap v156: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 2.7 MiB/s wr, 321 op/s; 2.3 MiB/s, 41 objects/s recovering 2024-09-21T12:33:52.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 198991 9.65 MB/sec execute 390 sec latency 20.017 ms 2024-09-21T12:33:53.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 199738 9.68 MB/sec execute 391 sec latency 29.834 ms 2024-09-21T12:33:54.590 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:54 smithi078 ceph-mon[63866]: pgmap v157: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.0 MiB/s wr, 255 op/s; 708 KiB/s, 30 objects/s recovering 2024-09-21T12:33:54.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 200311 9.68 MB/sec execute 392 sec latency 58.416 ms 2024-09-21T12:33:55.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:54 smithi148 ceph-mon[49387]: pgmap v157: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.0 MiB/s wr, 255 op/s; 708 KiB/s, 30 objects/s recovering 2024-09-21T12:33:55.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 200767 9.68 MB/sec execute 393 sec latency 33.158 ms 2024-09-21T12:33:56.609 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:56 smithi078 ceph-mon[63866]: pgmap v158: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 1.6 MiB/s wr, 209 op/s; 581 KiB/s, 24 objects/s recovering 2024-09-21T12:33:56.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 201233 9.69 MB/sec execute 394 sec latency 44.800 ms 2024-09-21T12:33:57.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:56 smithi148 ceph-mon[49387]: pgmap v158: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 1.6 MiB/s wr, 209 op/s; 581 KiB/s, 24 objects/s recovering 2024-09-21T12:33:57.825 INFO:teuthology.orchestra.run.smithi078.stdout: 5 201890 9.70 MB/sec execute 395 sec latency 45.020 ms 2024-09-21T12:33:58.786 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:33:58 smithi078 ceph-mon[63866]: pgmap v159: 97 pgs: 97 active+clean; 349 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.6 KiB/s rd, 6.2 MiB/s wr, 616 op/s; 486 KiB/s, 20 objects/s recovering 2024-09-21T12:33:58.825 INFO:teuthology.orchestra.run.smithi078.stdout: 5 202478 9.74 MB/sec execute 396 sec latency 31.479 ms 2024-09-21T12:33:59.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:33:58 smithi148 ceph-mon[49387]: pgmap v159: 97 pgs: 97 active+clean; 349 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.6 KiB/s rd, 6.2 MiB/s wr, 616 op/s; 486 KiB/s, 20 objects/s recovering 2024-09-21T12:33:59.825 INFO:teuthology.orchestra.run.smithi078.stdout: 5 203056 9.76 MB/sec execute 397 sec latency 21.287 ms 2024-09-21T12:34:00.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:00 smithi078 ceph-mon[63866]: pgmap v160: 97 pgs: 97 active+clean; 349 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.6 MiB/s wr, 559 op/s; 440 KiB/s, 18 objects/s recovering 2024-09-21T12:34:00.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:00.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:34:00.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:00.824 INFO:teuthology.orchestra.run.smithi078.stdout: 5 203448 9.76 MB/sec execute 398 sec latency 47.038 ms 2024-09-21T12:34:01.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:00 smithi148 ceph-mon[49387]: pgmap v160: 97 pgs: 97 active+clean; 349 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.6 MiB/s wr, 559 op/s; 440 KiB/s, 18 objects/s recovering 2024-09-21T12:34:01.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:01.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:34:01.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:01.827 INFO:teuthology.orchestra.run.smithi078.stdout: 5 203896 9.76 MB/sec execute 399 sec latency 30.156 ms 2024-09-21T12:34:02.825 INFO:teuthology.orchestra.run.smithi078.stdout: 5 204308 9.75 MB/sec execute 400 sec latency 48.953 ms 2024-09-21T12:34:02.850 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:02 smithi078 ceph-mon[63866]: pgmap v161: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 8.4 MiB/s wr, 805 op/s; 387 KiB/s, 16 objects/s recovering 2024-09-21T12:34:03.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:02 smithi148 ceph-mon[49387]: pgmap v161: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 8.4 MiB/s wr, 805 op/s; 387 KiB/s, 16 objects/s recovering 2024-09-21T12:34:03.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:03 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T12:34:03.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:03 smithi078 ceph-mon[63866]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T12:34:03.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:03 smithi078 ceph-mon[63866]: Upgrade: osd.3 is safe to restart 2024-09-21T12:34:03.825 INFO:teuthology.orchestra.run.smithi078.stdout: 5 204621 9.75 MB/sec execute 401 sec latency 52.483 ms 2024-09-21T12:34:04.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:03 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T12:34:04.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:03 smithi148 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T12:34:04.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:03 smithi148 ceph-mon[49387]: Upgrade: osd.3 is safe to restart 2024-09-21T12:34:04.622 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:04 smithi078 ceph-mon[63866]: pgmap v162: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.3 MiB/s wr, 669 op/s 2024-09-21T12:34:04.825 INFO:teuthology.orchestra.run.smithi078.stdout: 5 204988 9.75 MB/sec execute 402 sec latency 56.405 ms 2024-09-21T12:34:04.951 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:04 smithi148 ceph-mon[49387]: pgmap v162: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.3 MiB/s wr, 669 op/s 2024-09-21T12:34:05.825 INFO:teuthology.orchestra.run.smithi078.stdout: 5 205290 9.73 MB/sec execute 403 sec latency 65.725 ms 2024-09-21T12:34:06.416 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:06 smithi078 ceph-mon[63866]: Upgrade: Updating osd.3 2024-09-21T12:34:06.416 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:06 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:06.416 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:06 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T12:34:06.416 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:06 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:34:06.416 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:06 smithi078 ceph-mon[63866]: Deploying daemon osd.3 on smithi078 2024-09-21T12:34:06.416 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:06 smithi078 ceph-mon[63866]: pgmap v163: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 667 op/s 2024-09-21T12:34:06.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:06 smithi148 ceph-mon[49387]: Upgrade: Updating osd.3 2024-09-21T12:34:06.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:06 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:06.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:06 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T12:34:06.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:06 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:34:06.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:06 smithi148 ceph-mon[49387]: Deploying daemon osd.3 on smithi078 2024-09-21T12:34:06.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:06 smithi148 ceph-mon[49387]: pgmap v163: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 667 op/s 2024-09-21T12:34:06.825 INFO:teuthology.orchestra.run.smithi078.stdout: 5 205587 9.73 MB/sec execute 404 sec latency 44.297 ms 2024-09-21T12:34:07.189 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-21T12:34:07.825 INFO:teuthology.orchestra.run.smithi078.stdout: 5 205873 9.73 MB/sec execute 405 sec latency 62.798 ms 2024-09-21T12:34:08.259 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T12:34:08.259 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (6m) 28s ago 15m 19.6M - 0.27.0 11f11916f8cd e6e67ab07dec 2024-09-21T12:34:08.259 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (3m) 28s ago 15m 6895k - 19.3.0-5104-g44178494 b415816db8b4 46a10ea0b111 2024-09-21T12:34:08.259 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi148 smithi148 running (2m) 2m ago 14m 6895k - 19.3.0-5104-g44178494 b415816db8b4 23deb0615621 2024-09-21T12:34:08.259 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (6m) 28s ago 15m 54.1M - dad864ee21e9 2a140c36091b 2024-09-21T12:34:08.259 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.ytpzwh smithi078 running (9m) 28s ago 9m 38.0M - 17.2.0 e1d6a67b021e 5e9b109502a2 2024-09-21T12:34:08.259 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi148.uukezr smithi148 running (9m) 2m ago 9m 13.7M - 17.2.0 e1d6a67b021e b7e8aa0f3a6c 2024-09-21T12:34:08.259 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.ntwyan smithi078 *:8443,9283,8765 running (5m) 28s ago 16m 517M - 19.3.0-5104-g44178494 b415816db8b4 a61baedebf43 2024-09-21T12:34:08.259 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi148.oyiqmx smithi148 *:8443,9283,8765 running (4m) 2m ago 14m 454M - 19.3.0-5104-g44178494 b415816db8b4 0255535c6fdb 2024-09-21T12:34:08.259 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (4m) 28s ago 16m 54.6M 2048M 19.3.0-5104-g44178494 b415816db8b4 22b3097efbd8 2024-09-21T12:34:08.260 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi148 smithi148 running (3m) 2m ago 14m 36.1M 2048M 19.3.0-5104-g44178494 b415816db8b4 df8b08509136 2024-09-21T12:34:08.260 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.ydbhru smithi078 *:2049 running (8m) 28s ago 8m 278M - 4.0 e1d6a67b021e 7e5b0274ac38 2024-09-21T12:34:08.260 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi148.einuzt smithi148 *:2049 running (8m) 2m ago 8m 86.8M - 4.0 e1d6a67b021e 76a8b0eabede 2024-09-21T12:34:08.260 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (6m) 28s ago 15m 11.7M - 1.7.0 72c9c2088986 301adb4e9e22 2024-09-21T12:34:08.260 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi148 smithi148 *:9100 running (5m) 2m ago 14m 11.3M - 1.7.0 72c9c2088986 0cceff4bf342 2024-09-21T12:34:08.260 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (2m) 28s ago 13m 288M 1936M 19.3.0-5104-g44178494 b415816db8b4 7374add97534 2024-09-21T12:34:08.260 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (88s) 28s ago 13m 223M 1936M 19.3.0-5104-g44178494 b415816db8b4 14655a8da4e5 2024-09-21T12:34:08.260 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (37s) 28s ago 12m 12.0M 1936M 19.3.0-5104-g44178494 b415816db8b4 07ceda32f383 2024-09-21T12:34:08.260 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (12m) 28s ago 12m 603M 1936M 17.2.0 e1d6a67b021e ef942c8556d0 2024-09-21T12:34:08.260 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi148 running (12m) 2m ago 12m 493M 2704M 17.2.0 e1d6a67b021e 62eefdd572c0 2024-09-21T12:34:08.260 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi148 running (11m) 2m ago 11m 314M 2704M 17.2.0 e1d6a67b021e 7b44be78baca 2024-09-21T12:34:08.260 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi148 running (11m) 2m ago 11m 509M 2704M 17.2.0 e1d6a67b021e 3d23057910ea 2024-09-21T12:34:08.260 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi148 running (10m) 2m ago 10m 334M 2704M 17.2.0 e1d6a67b021e f9e4db9d35fa 2024-09-21T12:34:08.260 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (4m) 28s ago 14m 51.5M - 2.51.0 1d3b7f56885b face2d4cdd1b 2024-09-21T12:34:08.713 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:08 smithi078 ceph-mon[63866]: from='client.34300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:34:08.713 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:08 smithi078 ceph-mon[63866]: pgmap v164: 97 pgs: 97 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 9.2 MiB/s wr, 905 op/s 2024-09-21T12:34:08.713 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:08 smithi078 ceph-mon[63866]: from='client.34304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:34:08.825 INFO:teuthology.orchestra.run.smithi078.stdout: 5 206196 9.75 MB/sec execute 406 sec latency 42.207 ms 2024-09-21T12:34:08.829 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:34:08.830 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-21T12:34:08.830 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:34:08.830 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:34:08.830 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-21T12:34:08.830 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:34:08.830 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:34:08.830 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-21T12:34:08.830 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-21T12:34:08.830 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 3 2024-09-21T12:34:08.830 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:34:08.830 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-21T12:34:08.830 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:34:08.831 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:34:08.831 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-21T12:34:08.831 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-21T12:34:08.831 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 7 2024-09-21T12:34:08.831 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T12:34:08.831 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:34:09.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:08 smithi148 ceph-mon[49387]: from='client.34300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:34:09.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:08 smithi148 ceph-mon[49387]: pgmap v164: 97 pgs: 97 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 9.2 MiB/s wr, 905 op/s 2024-09-21T12:34:09.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:08 smithi148 ceph-mon[49387]: from='client.34304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:34:09.371 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:34:09.371 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5", 2024-09-21T12:34:09.371 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-21T12:34:09.371 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T12:34:09.371 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-21T12:34:09.371 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-21T12:34:09.372 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-09-21T12:34:09.372 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr" 2024-09-21T12:34:09.372 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-21T12:34:09.372 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "9/23 daemons upgraded", 2024-09-21T12:34:09.372 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T12:34:09.372 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-21T12:34:09.372 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:34:09.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:09 smithi078 ceph-mon[63866]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:34:09.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:09 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/2047110443' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:34:09.825 INFO:teuthology.orchestra.run.smithi078.stdout: 5 206478 9.75 MB/sec execute 407 sec latency 40.766 ms 2024-09-21T12:34:10.043 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-21T12:34:10.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:09 smithi148 ceph-mon[49387]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:34:10.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:09 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/2047110443' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:34:10.745 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:10 smithi078 ceph-mon[63866]: from='client.44235 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:34:10.745 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:10 smithi078 ceph-mon[63866]: pgmap v165: 97 pgs: 97 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 5.3 MiB/s wr, 554 op/s 2024-09-21T12:34:10.745 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:10 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/106653052' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:34:10.745 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:10 smithi078 systemd[1]: Stopping Ceph osd.3 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:34:10.825 INFO:teuthology.orchestra.run.smithi078.stdout: 5 206749 9.75 MB/sec execute 408 sec latency 80.387 ms 2024-09-21T12:34:10.996 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:10 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3[45093]: 2024-09-21T12:34:10.907+0000 7f1c587f8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T12:34:10.996 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:10 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3[45093]: 2024-09-21T12:34:10.908+0000 7f1c587f8700 -1 osd.3 109 *** Got signal Terminated *** 2024-09-21T12:34:10.996 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:10 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3[45093]: 2024-09-21T12:34:10.908+0000 7f1c587f8700 -1 osd.3 109 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T12:34:11.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:10 smithi148 ceph-mon[49387]: from='client.44235 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:34:11.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:10 smithi148 ceph-mon[49387]: pgmap v165: 97 pgs: 97 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 5.3 MiB/s wr, 554 op/s 2024-09-21T12:34:11.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:10 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/106653052' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:34:11.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:11 smithi078 ceph-mon[63866]: osd.3 marked itself down and dead 2024-09-21T12:34:11.825 INFO:teuthology.orchestra.run.smithi078.stdout: 5 206919 9.73 MB/sec execute 409 sec latency 259.553 ms 2024-09-21T12:34:12.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:11 smithi148 ceph-mon[49387]: osd.3 marked itself down and dead 2024-09-21T12:34:12.493 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:12 smithi078 podman[75724]: 2024-09-21 12:34:12.230085123 +0000 UTC m=+1.512594408 container died ef942c8556d0ef5ac381b54a9b25dbd4aed67e863fadfb6a102e76e7d2e5b4a9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3, vendor=Red Hat, Inc., release=754, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, architecture=x86_64, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, RELEASE=HEAD, com.redhat.component=centos-stream-container, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-21T12:34:12.744 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:12 smithi078 ceph-mon[63866]: pgmap v166: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 7.5 MiB/s wr, 722 op/s 2024-09-21T12:34:12.744 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:12 smithi078 ceph-mon[63866]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T12:34:12.744 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:12 smithi078 ceph-mon[63866]: osdmap e110: 8 total, 7 up, 8 in 2024-09-21T12:34:12.825 INFO:teuthology.orchestra.run.smithi078.stdout: 5 206968 9.71 MB/sec execute 410 sec latency 1057.451 ms 2024-09-21T12:34:12.996 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:12 smithi078 podman[75724]: 2024-09-21 12:34:12.78550112 +0000 UTC m=+2.068010397 container cleanup ef942c8556d0ef5ac381b54a9b25dbd4aed67e863fadfb6a102e76e7d2e5b4a9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3, name=centos-stream, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, version=8, release=754, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, maintainer=Guillaume Abrioux , vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, vendor=Red Hat, Inc., distribution-scope=public, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-21T12:34:12.996 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:12 smithi078 bash[75724]: ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3 2024-09-21T12:34:13.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:12 smithi148 ceph-mon[49387]: pgmap v166: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 7.5 MiB/s wr, 722 op/s 2024-09-21T12:34:13.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:12 smithi148 ceph-mon[49387]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T12:34:13.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:12 smithi148 ceph-mon[49387]: osdmap e110: 8 total, 7 up, 8 in 2024-09-21T12:34:13.249 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:13 smithi078 podman[75756]: 2024-09-21 12:34:13.216312954 +0000 UTC m=+0.978710499 container remove ef942c8556d0ef5ac381b54a9b25dbd4aed67e863fadfb6a102e76e7d2e5b4a9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, distribution-scope=public, release=754, vendor=Red Hat, Inc., architecture=x86_64, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, com.redhat.component=centos-stream-container, io.openshift.expose-services=, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8) 2024-09-21T12:34:13.755 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:13 smithi078 ceph-mon[63866]: osdmap e111: 8 total, 7 up, 8 in 2024-09-21T12:34:13.755 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:13 smithi078 podman[75841]: 2024-09-21 12:34:13.493575839 +0000 UTC m=+0.022005487 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:34:13.755 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:13 smithi078 podman[75841]: 2024-09-21 12:34:13.670977707 +0000 UTC m=+0.199407346 container create 031035b72a24b1d795c496a23b928459134407694082c6fd5c9e51ea392a7643 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3-deactivate, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T12:34:13.825 INFO:teuthology.orchestra.run.smithi078.stdout: 5 207356 9.70 MB/sec execute 411 sec latency 36.345 ms 2024-09-21T12:34:14.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:13 smithi148 ceph-mon[49387]: osdmap e111: 8 total, 7 up, 8 in 2024-09-21T12:34:14.264 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:14 smithi078 podman[75841]: 2024-09-21 12:34:14.124433414 +0000 UTC m=+0.652863064 container init 031035b72a24b1d795c496a23b928459134407694082c6fd5c9e51ea392a7643 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T12:34:14.264 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:14 smithi078 podman[75841]: 2024-09-21 12:34:14.141774699 +0000 UTC m=+0.670204343 container start 031035b72a24b1d795c496a23b928459134407694082c6fd5c9e51ea392a7643 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3-deactivate, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T12:34:14.515 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:14 smithi078 podman[75841]: 2024-09-21 12:34:14.316652972 +0000 UTC m=+0.845082611 container attach 031035b72a24b1d795c496a23b928459134407694082c6fd5c9e51ea392a7643 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0842-4417849, maintainer=Guillaume Abrioux ) 2024-09-21T12:34:14.515 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:14 smithi078 podman[75841]: 2024-09-21 12:34:14.441357638 +0000 UTC m=+0.969787284 container died 031035b72a24b1d795c496a23b928459134407694082c6fd5c9e51ea392a7643 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD) 2024-09-21T12:34:14.765 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:14 smithi078 ceph-mon[63866]: pgmap v169: 97 pgs: 1 active+undersized, 15 stale+active+clean, 3 active+undersized+degraded, 78 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 6.1 MiB/s wr, 615 op/s; 11/1701 objects degraded (0.647%) 2024-09-21T12:34:14.765 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:14 smithi078 ceph-mon[63866]: Health check failed: Degraded data redundancy: 11/1701 objects degraded (0.647%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T12:34:14.825 INFO:teuthology.orchestra.run.smithi078.stdout: 5 207761 9.70 MB/sec execute 412 sec latency 32.175 ms 2024-09-21T12:34:15.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:14 smithi148 ceph-mon[49387]: pgmap v169: 97 pgs: 1 active+undersized, 15 stale+active+clean, 3 active+undersized+degraded, 78 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 6.1 MiB/s wr, 615 op/s; 11/1701 objects degraded (0.647%) 2024-09-21T12:34:15.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:14 smithi148 ceph-mon[49387]: Health check failed: Degraded data redundancy: 11/1701 objects degraded (0.647%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T12:34:15.799 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:15 smithi078 podman[75863]: 2024-09-21 12:34:15.64908402 +0000 UTC m=+1.199524812 container remove 031035b72a24b1d795c496a23b928459134407694082c6fd5c9e51ea392a7643 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-21-0842-4417849, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T12:34:15.799 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:15 smithi078 systemd[1]: ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.3.service: Deactivated successfully. 2024-09-21T12:34:15.799 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:15 smithi078 systemd[1]: Stopped Ceph osd.3 for 6263deca-7813-11ef-baf6-efdc52797490. 2024-09-21T12:34:15.799 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:15 smithi078 systemd[1]: ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.3.service: Consumed 3min 52.033s CPU time. 2024-09-21T12:34:15.825 INFO:teuthology.orchestra.run.smithi078.stdout: 5 208212 9.70 MB/sec execute 413 sec latency 45.029 ms 2024-09-21T12:34:16.050 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:16 smithi078 systemd[1]: Starting Ceph osd.3 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:34:16.050 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:15 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:16.050 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:15 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:34:16.050 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:15 smithi078 ceph-mon[63866]: pgmap v170: 97 pgs: 1 active+undersized, 15 stale+active+clean, 3 active+undersized+degraded, 78 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.3 MiB/s wr, 259 op/s; 11/1701 objects degraded (0.647%) 2024-09-21T12:34:16.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:15 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:16.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:15 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:34:16.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:15 smithi148 ceph-mon[49387]: pgmap v170: 97 pgs: 1 active+undersized, 15 stale+active+clean, 3 active+undersized+degraded, 78 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.3 MiB/s wr, 259 op/s; 11/1701 objects degraded (0.647%) 2024-09-21T12:34:16.553 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:16 smithi078 podman[75956]: 2024-09-21 12:34:16.210969154 +0000 UTC m=+0.022198995 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:34:16.553 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:16 smithi078 podman[75956]: 2024-09-21 12:34:16.360624629 +0000 UTC m=+0.171854461 container create a917c9f99d4c3c90a6ebbfac672e50287bf9ee7f2387cce961ce4339935341fe (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3-activate, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T12:34:16.804 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:16 smithi078 podman[75956]: 2024-09-21 12:34:16.75579948 +0000 UTC m=+0.567029332 container init a917c9f99d4c3c90a6ebbfac672e50287bf9ee7f2387cce961ce4339935341fe (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0842-4417849, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-21T12:34:16.804 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:16 smithi078 podman[75956]: 2024-09-21 12:34:16.769293202 +0000 UTC m=+0.580523030 container start a917c9f99d4c3c90a6ebbfac672e50287bf9ee7f2387cce961ce4339935341fe (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3-activate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0) 2024-09-21T12:34:16.825 INFO:teuthology.orchestra.run.smithi078.stdout: 5 208666 9.70 MB/sec execute 414 sec latency 55.807 ms 2024-09-21T12:34:17.055 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:16 smithi078 podman[75956]: 2024-09-21 12:34:16.91439532 +0000 UTC m=+0.725625167 container attach a917c9f99d4c3c90a6ebbfac672e50287bf9ee7f2387cce961ce4339935341fe (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3-activate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-21-0842-4417849) 2024-09-21T12:34:17.055 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:16 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3-activate[75969]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:34:17.055 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:16 smithi078 bash[75956]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:34:17.055 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:16 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3-activate[75969]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:34:17.055 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:16 smithi078 bash[75956]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:34:17.826 INFO:teuthology.orchestra.run.smithi078.stdout: 5 209126 9.70 MB/sec execute 415 sec latency 29.189 ms 2024-09-21T12:34:18.593 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:18 smithi078 ceph-mon[63866]: pgmap v171: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.7 MiB/s wr, 536 op/s; 216/1746 objects degraded (12.371%) 2024-09-21T12:34:18.826 INFO:teuthology.orchestra.run.smithi078.stdout: 5 209546 9.72 MB/sec execute 416 sec latency 38.341 ms 2024-09-21T12:34:19.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:18 smithi148 ceph-mon[49387]: pgmap v171: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.7 MiB/s wr, 536 op/s; 216/1746 objects degraded (12.371%) 2024-09-21T12:34:19.826 INFO:teuthology.orchestra.run.smithi078.stdout: 5 210000 9.74 MB/sec execute 417 sec latency 33.767 ms 2024-09-21T12:34:20.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:20 smithi078 ceph-mon[63866]: pgmap v172: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 2.5 MiB/s wr, 283 op/s; 216/1746 objects degraded (12.371%) 2024-09-21T12:34:20.826 INFO:teuthology.orchestra.run.smithi078.stdout: 5 210366 9.75 MB/sec execute 418 sec latency 41.758 ms 2024-09-21T12:34:21.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:20 smithi148 ceph-mon[49387]: pgmap v172: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 2.5 MiB/s wr, 283 op/s; 216/1746 objects degraded (12.371%) 2024-09-21T12:34:21.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:21 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 204/1734 objects degraded (11.765%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T12:34:21.646 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:21 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3-activate[75969]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T12:34:21.646 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:21 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3-activate[75969]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:34:21.646 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:21 smithi078 bash[75956]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T12:34:21.646 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:21 smithi078 bash[75956]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:34:21.646 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:21 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3-activate[75969]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:34:21.646 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:21 smithi078 bash[75956]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:34:21.826 INFO:teuthology.orchestra.run.smithi078.stdout: 5 210716 9.73 MB/sec execute 419 sec latency 30.103 ms 2024-09-21T12:34:21.897 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:21 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3-activate[75969]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T12:34:21.897 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:21 smithi078 bash[75956]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T12:34:21.897 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:21 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3-activate[75969]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-09-21T12:34:21.897 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:21 smithi078 bash[75956]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-09-21T12:34:22.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:21 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 204/1734 objects degraded (11.765%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T12:34:22.403 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:22 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3-activate[75969]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-21T12:34:22.403 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:22 smithi078 bash[75956]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-21T12:34:22.403 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:22 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3-activate[75969]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-09-21T12:34:22.404 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:22 smithi078 bash[75956]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-09-21T12:34:22.404 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:22 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3-activate[75969]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T12:34:22.404 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:22 smithi078 bash[75956]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T12:34:22.404 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:22 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3-activate[75969]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T12:34:22.404 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:22 smithi078 bash[75956]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T12:34:22.404 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:22 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3-activate[75969]: --> ceph-volume lvm activate successful for osd ID: 3 2024-09-21T12:34:22.404 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:22 smithi078 bash[75956]: --> ceph-volume lvm activate successful for osd ID: 3 2024-09-21T12:34:22.404 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:22 smithi078 podman[75956]: 2024-09-21 12:34:22.252308916 +0000 UTC m=+6.063538772 container died a917c9f99d4c3c90a6ebbfac672e50287bf9ee7f2387cce961ce4339935341fe (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3-activate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T12:34:22.673 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:22 smithi078 ceph-mon[63866]: pgmap v173: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 5.4 MiB/s wr, 535 op/s; 204/1734 objects degraded (11.765%) 2024-09-21T12:34:22.826 INFO:teuthology.orchestra.run.smithi078.stdout: 5 211090 9.72 MB/sec execute 420 sec latency 50.206 ms 2024-09-21T12:34:23.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:22 smithi148 ceph-mon[49387]: pgmap v173: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 5.4 MiB/s wr, 535 op/s; 204/1734 objects degraded (11.765%) 2024-09-21T12:34:23.700 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:23 smithi078 podman[76201]: 2024-09-21 12:34:23.447930953 +0000 UTC m=+1.177669427 container remove a917c9f99d4c3c90a6ebbfac672e50287bf9ee7f2387cce961ce4339935341fe (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3-activate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T12:34:23.700 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:23 smithi078 podman[76257]: 2024-09-21 12:34:23.584181953 +0000 UTC m=+0.020224052 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:34:23.700 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:23 smithi078 podman[76257]: 2024-09-21 12:34:23.699138134 +0000 UTC m=+0.135180199 container create ead87e4306ff0b3cc82222ec6c523a2353649bf817838ad05d2c20f0a263e36c (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=) 2024-09-21T12:34:23.826 INFO:teuthology.orchestra.run.smithi078.stdout: 5 211375 9.72 MB/sec execute 421 sec latency 52.227 ms 2024-09-21T12:34:24.227 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:24 smithi078 podman[76257]: 2024-09-21 12:34:24.169984618 +0000 UTC m=+0.606026695 container init ead87e4306ff0b3cc82222ec6c523a2353649bf817838ad05d2c20f0a263e36c (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-21T12:34:24.227 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:24 smithi078 podman[76257]: 2024-09-21 12:34:24.184552972 +0000 UTC m=+0.620595047 container start ead87e4306ff0b3cc82222ec6c523a2353649bf817838ad05d2c20f0a263e36c (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0842-4417849, maintainer=Guillaume Abrioux ) 2024-09-21T12:34:24.227 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:24 smithi078 ceph-osd[76274]: -- 172.21.15.78:0/1868348094 <== mon.0 v2:172.21.15.78:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x564f8bf101e0 con 0x564f8b0afc00 2024-09-21T12:34:24.478 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:24 smithi078 bash[76257]: ead87e4306ff0b3cc82222ec6c523a2353649bf817838ad05d2c20f0a263e36c 2024-09-21T12:34:24.478 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:24 smithi078 systemd[1]: Started Ceph osd.3 for 6263deca-7813-11ef-baf6-efdc52797490. 2024-09-21T12:34:24.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:24 smithi078 ceph-mon[63866]: pgmap v174: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.0 MiB/s wr, 488 op/s; 204/1731 objects degraded (11.785%) 2024-09-21T12:34:24.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:24 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:24.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:24 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:24.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:24 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:34:24.826 INFO:teuthology.orchestra.run.smithi078.stdout: 5 211765 9.72 MB/sec execute 422 sec latency 33.267 ms 2024-09-21T12:34:25.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:24 smithi148 ceph-mon[49387]: pgmap v174: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.0 MiB/s wr, 488 op/s; 204/1731 objects degraded (11.785%) 2024-09-21T12:34:25.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:24 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:25.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:24 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:25.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:24 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:34:25.826 INFO:teuthology.orchestra.run.smithi078.stdout: 5 212150 9.72 MB/sec execute 423 sec latency 64.182 ms 2024-09-21T12:34:26.496 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:26 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3[76270]: 2024-09-21T12:34:26.324+0000 7fc364638640 -1 Falling back to public interface 2024-09-21T12:34:26.747 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:26 smithi078 ceph-mon[63866]: pgmap v175: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.5 MiB/s wr, 443 op/s; 204/1731 objects degraded (11.785%) 2024-09-21T12:34:26.826 INFO:teuthology.orchestra.run.smithi078.stdout: 5 212638 9.72 MB/sec execute 424 sec latency 62.983 ms 2024-09-21T12:34:27.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:26 smithi148 ceph-mon[49387]: pgmap v175: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.5 MiB/s wr, 443 op/s; 204/1731 objects degraded (11.785%) 2024-09-21T12:34:27.738 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:27 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 204/1731 objects degraded (11.785%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T12:34:27.826 INFO:teuthology.orchestra.run.smithi078.stdout: 5 213114 9.74 MB/sec execute 425 sec latency 57.936 ms 2024-09-21T12:34:28.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:27 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 204/1731 objects degraded (11.785%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T12:34:28.747 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:28 smithi078 ceph-mon[63866]: pgmap v176: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 6.4 MiB/s wr, 678 op/s; 201/1584 objects degraded (12.689%) 2024-09-21T12:34:28.826 INFO:teuthology.orchestra.run.smithi078.stdout: 5 213764 9.77 MB/sec execute 426 sec latency 28.757 ms 2024-09-21T12:34:29.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:28 smithi148 ceph-mon[49387]: pgmap v176: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 6.4 MiB/s wr, 678 op/s; 201/1584 objects degraded (12.689%) 2024-09-21T12:34:29.826 INFO:teuthology.orchestra.run.smithi078.stdout: 5 214133 9.77 MB/sec execute 427 sec latency 28.676 ms 2024-09-21T12:34:30.766 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:30 smithi078 ceph-mon[63866]: pgmap v177: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.7 MiB/s wr, 494 op/s; 201/1584 objects degraded (12.689%) 2024-09-21T12:34:30.766 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:34:30.826 INFO:teuthology.orchestra.run.smithi078.stdout: 5 214464 9.76 MB/sec execute 428 sec latency 50.034 ms 2024-09-21T12:34:31.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:30 smithi148 ceph-mon[49387]: pgmap v177: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.7 MiB/s wr, 494 op/s; 201/1584 objects degraded (12.689%) 2024-09-21T12:34:31.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:34:31.826 INFO:teuthology.orchestra.run.smithi078.stdout: 5 214793 9.75 MB/sec execute 429 sec latency 58.695 ms 2024-09-21T12:34:32.793 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:32 smithi078 ceph-mon[63866]: pgmap v178: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 7.9 MiB/s wr, 758 op/s; 216/1737 objects degraded (12.435%) 2024-09-21T12:34:32.793 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:32 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 216/1737 objects degraded (12.435%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T12:34:32.827 INFO:teuthology.orchestra.run.smithi078.stdout: 5 215400 9.77 MB/sec execute 430 sec latency 45.146 ms 2024-09-21T12:34:33.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:32 smithi148 ceph-mon[49387]: pgmap v178: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 7.9 MiB/s wr, 758 op/s; 216/1737 objects degraded (12.435%) 2024-09-21T12:34:33.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:32 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 216/1737 objects degraded (12.435%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T12:34:33.827 INFO:teuthology.orchestra.run.smithi078.stdout: 5 215827 9.76 MB/sec execute 431 sec latency 52.857 ms 2024-09-21T12:34:34.827 INFO:teuthology.orchestra.run.smithi078.stdout: 5 216259 9.76 MB/sec execute 432 sec latency 35.942 ms 2024-09-21T12:34:34.863 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:34 smithi078 ceph-mon[63866]: pgmap v179: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.0 MiB/s wr, 506 op/s; 216/1734 objects degraded (12.457%) 2024-09-21T12:34:34.863 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:34 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:34.863 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:34 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:35.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:34 smithi148 ceph-mon[49387]: pgmap v179: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.0 MiB/s wr, 506 op/s; 216/1734 objects degraded (12.457%) 2024-09-21T12:34:35.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:34 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:35.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:34 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:35.118 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:35 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3[76270]: 2024-09-21T12:34:35.083+0000 7fc364638640 -1 osd.3 0 read_superblock omap replica is missing. 2024-09-21T12:34:35.827 INFO:teuthology.orchestra.run.smithi078.stdout: 5 216605 9.78 MB/sec execute 433 sec latency 44.224 ms 2024-09-21T12:34:35.891 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:35 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:35.891 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:35 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:36.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:35 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:36.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:35 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:36.397 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:36 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3[76270]: 2024-09-21T12:34:36.273+0000 7fc364638640 -1 osd.3 109 log_to_monitors true 2024-09-21T12:34:36.827 INFO:teuthology.orchestra.run.smithi078.stdout: 5 216941 9.78 MB/sec execute 434 sec latency 87.322 ms 2024-09-21T12:34:36.902 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:36 smithi078 ceph-mon[63866]: pgmap v180: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.4 MiB/s wr, 543 op/s; 215/1704 objects degraded (12.617%) 2024-09-21T12:34:36.903 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:36 smithi078 ceph-mon[63866]: from='osd.3 [v2:172.21.15.78:6826/2111605658,v1:172.21.15.78:6827/2111605658]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T12:34:37.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:36 smithi148 ceph-mon[49387]: pgmap v180: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.4 MiB/s wr, 543 op/s; 215/1704 objects degraded (12.617%) 2024-09-21T12:34:37.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:36 smithi148 ceph-mon[49387]: from='osd.3 [v2:172.21.15.78:6826/2111605658,v1:172.21.15.78:6827/2111605658]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T12:34:37.456 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:34:37 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3[76270]: 2024-09-21T12:34:37.406+0000 7fc3583da640 -1 osd.3 109 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T12:34:37.827 INFO:teuthology.orchestra.run.smithi078.stdout: 5 217122 9.77 MB/sec execute 435 sec latency 115.928 ms 2024-09-21T12:34:37.986 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:37 smithi078 ceph-mon[63866]: from='osd.3 [v2:172.21.15.78:6826/2111605658,v1:172.21.15.78:6827/2111605658]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T12:34:37.986 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:37 smithi078 ceph-mon[63866]: osdmap e112: 8 total, 7 up, 8 in 2024-09-21T12:34:37.986 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:37 smithi078 ceph-mon[63866]: from='osd.3 [v2:172.21.15.78:6826/2111605658,v1:172.21.15.78:6827/2111605658]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]} : dispatch 2024-09-21T12:34:37.986 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:37 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 215/1704 objects degraded (12.617%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T12:34:37.986 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:37 smithi078 ceph-mon[63866]: from='osd.3 [v2:172.21.15.78:6826/2111605658,v1:172.21.15.78:6827/2111605658]' entity='osd.3' 2024-09-21T12:34:37.986 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:37 smithi078 ceph-mon[63866]: pgmap v182: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.5 MiB/s wr, 621 op/s; 200/1644 objects degraded (12.165%) 2024-09-21T12:34:38.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:37 smithi148 ceph-mon[49387]: from='osd.3 [v2:172.21.15.78:6826/2111605658,v1:172.21.15.78:6827/2111605658]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T12:34:38.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:37 smithi148 ceph-mon[49387]: osdmap e112: 8 total, 7 up, 8 in 2024-09-21T12:34:38.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:37 smithi148 ceph-mon[49387]: from='osd.3 [v2:172.21.15.78:6826/2111605658,v1:172.21.15.78:6827/2111605658]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi078", "root=default"]} : dispatch 2024-09-21T12:34:38.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:37 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 215/1704 objects degraded (12.617%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T12:34:38.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:37 smithi148 ceph-mon[49387]: from='osd.3 [v2:172.21.15.78:6826/2111605658,v1:172.21.15.78:6827/2111605658]' entity='osd.3' 2024-09-21T12:34:38.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:37 smithi148 ceph-mon[49387]: pgmap v182: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.5 MiB/s wr, 621 op/s; 200/1644 objects degraded (12.165%) 2024-09-21T12:34:38.827 INFO:teuthology.orchestra.run.smithi078.stdout: 5 217417 9.78 MB/sec execute 436 sec latency 438.442 ms 2024-09-21T12:34:39.023 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:38 smithi078 ceph-mon[63866]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T12:34:39.023 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:38 smithi078 ceph-mon[63866]: osd.3 [v2:172.21.15.78:6826/2111605658,v1:172.21.15.78:6827/2111605658] boot 2024-09-21T12:34:39.023 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:38 smithi078 ceph-mon[63866]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T12:34:39.023 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:38 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T12:34:39.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:38 smithi148 ceph-mon[49387]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T12:34:39.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:38 smithi148 ceph-mon[49387]: osd.3 [v2:172.21.15.78:6826/2111605658,v1:172.21.15.78:6827/2111605658] boot 2024-09-21T12:34:39.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:38 smithi148 ceph-mon[49387]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T12:34:39.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:38 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T12:34:39.827 INFO:teuthology.orchestra.run.smithi078.stdout: 5 217480 9.75 MB/sec execute 437 sec latency 1438.471 ms 2024-09-21T12:34:40.436 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-21T12:34:40.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:40 smithi148 ceph-mon[49387]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T12:34:40.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:40 smithi148 ceph-mon[49387]: pgmap v185: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.5 MiB/s wr, 501 op/s; 200/1641 objects degraded (12.188%) 2024-09-21T12:34:40.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:40 smithi078 ceph-mon[63866]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T12:34:40.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:40 smithi078 ceph-mon[63866]: pgmap v185: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.5 MiB/s wr, 501 op/s; 200/1641 objects degraded (12.188%) 2024-09-21T12:34:40.827 INFO:teuthology.orchestra.run.smithi078.stdout: 5 217480 9.73 MB/sec execute 438 sec latency 2438.496 ms 2024-09-21T12:34:41.396 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T12:34:41.397 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (6m) 7s ago 15m 20.0M - 0.27.0 11f11916f8cd e6e67ab07dec 2024-09-21T12:34:41.397 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (3m) 7s ago 15m 6895k - 19.3.0-5104-g44178494 b415816db8b4 46a10ea0b111 2024-09-21T12:34:41.397 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi148 smithi148 running (3m) 3m ago 15m 6895k - 19.3.0-5104-g44178494 b415816db8b4 23deb0615621 2024-09-21T12:34:41.397 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (6m) 7s ago 15m 53.3M - dad864ee21e9 2a140c36091b 2024-09-21T12:34:41.397 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.ytpzwh smithi078 running (9m) 7s ago 9m 38.4M - 17.2.0 e1d6a67b021e 5e9b109502a2 2024-09-21T12:34:41.397 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi148.uukezr smithi148 running (9m) 3m ago 9m 13.7M - 17.2.0 e1d6a67b021e b7e8aa0f3a6c 2024-09-21T12:34:41.397 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.ntwyan smithi078 *:8443,9283,8765 running (5m) 7s ago 17m 519M - 19.3.0-5104-g44178494 b415816db8b4 a61baedebf43 2024-09-21T12:34:41.397 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi148.oyiqmx smithi148 *:8443,9283,8765 running (5m) 3m ago 15m 454M - 19.3.0-5104-g44178494 b415816db8b4 0255535c6fdb 2024-09-21T12:34:41.397 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (4m) 7s ago 17m 55.9M 2048M 19.3.0-5104-g44178494 b415816db8b4 22b3097efbd8 2024-09-21T12:34:41.397 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi148 smithi148 running (4m) 3m ago 15m 36.1M 2048M 19.3.0-5104-g44178494 b415816db8b4 df8b08509136 2024-09-21T12:34:41.397 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.ydbhru smithi078 *:2049 running (9m) 7s ago 9m 279M - 4.0 e1d6a67b021e 7e5b0274ac38 2024-09-21T12:34:41.397 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi148.einuzt smithi148 *:2049 running (9m) 3m ago 9m 86.8M - 4.0 e1d6a67b021e 76a8b0eabede 2024-09-21T12:34:41.398 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (6m) 7s ago 15m 11.0M - 1.7.0 72c9c2088986 301adb4e9e22 2024-09-21T12:34:41.398 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi148 smithi148 *:9100 running (6m) 3m ago 14m 11.3M - 1.7.0 72c9c2088986 0cceff4bf342 2024-09-21T12:34:41.398 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (2m) 7s ago 14m 345M 1936M 19.3.0-5104-g44178494 b415816db8b4 7374add97534 2024-09-21T12:34:41.398 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (2m) 7s ago 13m 276M 1936M 19.3.0-5104-g44178494 b415816db8b4 14655a8da4e5 2024-09-21T12:34:41.398 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (71s) 7s ago 13m 219M 1936M 19.3.0-5104-g44178494 b415816db8b4 07ceda32f383 2024-09-21T12:34:41.398 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (17s) 7s ago 12m 11.4M 1936M 19.3.0-5104-g44178494 b415816db8b4 ead87e4306ff 2024-09-21T12:34:41.398 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi148 running (12m) 3m ago 12m 493M 2704M 17.2.0 e1d6a67b021e 62eefdd572c0 2024-09-21T12:34:41.398 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi148 running (12m) 3m ago 12m 314M 2704M 17.2.0 e1d6a67b021e 7b44be78baca 2024-09-21T12:34:41.398 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi148 running (11m) 3m ago 11m 509M 2704M 17.2.0 e1d6a67b021e 3d23057910ea 2024-09-21T12:34:41.398 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi148 running (11m) 3m ago 11m 334M 2704M 17.2.0 e1d6a67b021e f9e4db9d35fa 2024-09-21T12:34:41.398 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (5m) 7s ago 14m 55.8M - 2.51.0 1d3b7f56885b face2d4cdd1b 2024-09-21T12:34:41.420 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:41 smithi078 ceph-mon[63866]: from='client.34328 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:34:41.421 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:41 smithi078 ceph-mon[63866]: osdmap e115: 8 total, 8 up, 8 in 2024-09-21T12:34:41.421 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:41 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:41.421 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:41 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:41.421 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:41 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:34:41.421 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:41 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:34:41.421 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:41 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:41.421 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:41 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:41.421 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:41 smithi078 ceph-mon[63866]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:34:41.421 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:41 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:34:41.421 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:41 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:34:41.422 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:41 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:34:41.422 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:41 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:34:41.422 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:41 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-21T12:34:41.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:41 smithi148 ceph-mon[49387]: from='client.34328 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:34:41.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:41 smithi148 ceph-mon[49387]: osdmap e115: 8 total, 8 up, 8 in 2024-09-21T12:34:41.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:41 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:41.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:41 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:41.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:41 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:34:41.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:41 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:34:41.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:41 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:41.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:41 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:41.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:41 smithi148 ceph-mon[49387]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:34:41.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:41 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:34:41.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:41 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:34:41.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:41 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:34:41.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:41 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:34:41.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:41 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-21T12:34:41.827 INFO:teuthology.orchestra.run.smithi078.stdout: 5 217669 9.72 MB/sec execute 439 sec latency 2544.444 ms 2024-09-21T12:34:42.057 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:34:42.057 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-21T12:34:42.058 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:34:42.058 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:34:42.058 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-21T12:34:42.058 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:34:42.058 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:34:42.058 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-21T12:34:42.058 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-21T12:34:42.058 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 4 2024-09-21T12:34:42.059 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:34:42.059 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-21T12:34:42.059 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:34:42.059 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:34:42.059 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-21T12:34:42.059 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-21T12:34:42.059 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 8 2024-09-21T12:34:42.059 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T12:34:42.059 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:34:42.617 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:34:42.617 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5", 2024-09-21T12:34:42.617 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-21T12:34:42.617 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T12:34:42.617 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-21T12:34:42.617 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-21T12:34:42.617 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-09-21T12:34:42.617 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr" 2024-09-21T12:34:42.617 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-21T12:34:42.618 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "10/23 daemons upgraded", 2024-09-21T12:34:42.618 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T12:34:42.618 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-21T12:34:42.618 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:34:42.700 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:42 smithi078 ceph-mon[63866]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T12:34:42.701 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:42 smithi078 ceph-mon[63866]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-21T12:34:42.701 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:42 smithi078 ceph-mon[63866]: from='client.34336 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:34:42.701 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:42 smithi078 ceph-mon[63866]: pgmap v187: 97 pgs: 8 activating+undersized+degraded+remapped, 7 active+undersized, 15 active+undersized+degraded, 67 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 2.7 MiB/s wr, 255 op/s; 293/1671 objects degraded (17.534%); 19 B/s, 0 objects/s recovering 2024-09-21T12:34:42.701 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:42 smithi078 ceph-mon[63866]: osdmap e116: 8 total, 8 up, 8 in 2024-09-21T12:34:42.701 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:42 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 200/1641 objects degraded (12.188%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T12:34:42.701 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:42 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/3571422276' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:34:42.827 INFO:teuthology.orchestra.run.smithi078.stdout: 5 217920 9.71 MB/sec execute 440 sec latency 188.141 ms 2024-09-21T12:34:43.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:42 smithi148 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T12:34:43.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:42 smithi148 ceph-mon[49387]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-21T12:34:43.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:42 smithi148 ceph-mon[49387]: from='client.34336 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:34:43.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:42 smithi148 ceph-mon[49387]: pgmap v187: 97 pgs: 8 activating+undersized+degraded+remapped, 7 active+undersized, 15 active+undersized+degraded, 67 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 2.7 MiB/s wr, 255 op/s; 293/1671 objects degraded (17.534%); 19 B/s, 0 objects/s recovering 2024-09-21T12:34:43.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:42 smithi148 ceph-mon[49387]: osdmap e116: 8 total, 8 up, 8 in 2024-09-21T12:34:43.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:42 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 200/1641 objects degraded (12.188%), 27 pgs degraded (PG_DEGRADED) 2024-09-21T12:34:43.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:42 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/3571422276' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:34:43.384 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_WARN Degraded data redundancy: 293/1671 objects degraded (17.534%), 23 pgs degraded 2024-09-21T12:34:43.384 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 293/1671 objects degraded (17.534%), 23 pgs degraded 2024-09-21T12:34:43.384 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1 is active+undersized+degraded, acting [2,0] 2024-09-21T12:34:43.384 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.d is activating+undersized+degraded+remapped, acting [1,4] 2024-09-21T12:34:43.384 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.e is active+undersized+degraded, acting [2,7] 2024-09-21T12:34:43.384 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.13 is active+undersized+degraded, acting [0,4] 2024-09-21T12:34:43.384 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.19 is active+undersized+degraded, acting [6,0] 2024-09-21T12:34:43.384 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1b is activating+undersized+degraded+remapped, acting [6,7] 2024-09-21T12:34:43.385 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1e is active+undersized+degraded, acting [0,5] 2024-09-21T12:34:43.385 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1 is active+undersized+degraded, acting [0,4] 2024-09-21T12:34:43.385 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.2 is activating+undersized+degraded+remapped, acting [5,6] 2024-09-21T12:34:43.385 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.5 is activating+undersized+degraded+remapped, acting [5,2] 2024-09-21T12:34:43.385 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.7 is active+undersized+degraded, acting [7,0] 2024-09-21T12:34:43.385 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.8 is activating+undersized+degraded+remapped, acting [1,7] 2024-09-21T12:34:43.385 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.b is active+undersized+degraded, acting [0,4] 2024-09-21T12:34:43.385 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.c is activating+undersized+degraded+remapped, acting [5,6] 2024-09-21T12:34:43.385 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.12 is active+undersized+degraded, acting [0,7] 2024-09-21T12:34:43.385 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.15 is activating+undersized+degraded+remapped, acting [7,4] 2024-09-21T12:34:43.385 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.17 is active+undersized+degraded, acting [0,5] 2024-09-21T12:34:43.385 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.18 is active+undersized+degraded, acting [0,1] 2024-09-21T12:34:43.385 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.19 is activating+undersized+degraded+remapped, acting [1,4] 2024-09-21T12:34:43.386 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1e is active+undersized+degraded, acting [6,2] 2024-09-21T12:34:43.386 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.0 is active+undersized+degraded, acting [7,0] 2024-09-21T12:34:43.386 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.14 is active+undersized+degraded, acting [1,7] 2024-09-21T12:34:43.386 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.19 is active+undersized+degraded, acting [2,0] 2024-09-21T12:34:43.827 INFO:teuthology.orchestra.run.smithi078.stdout: 5 218079 9.69 MB/sec execute 441 sec latency 1015.559 ms 2024-09-21T12:34:43.887 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:43 smithi078 ceph-mon[63866]: from='client.34344 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:34:43.888 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:43 smithi078 ceph-mon[63866]: osdmap e117: 8 total, 8 up, 8 in 2024-09-21T12:34:43.888 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:43 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/1981107811' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:34:44.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:43 smithi148 ceph-mon[49387]: from='client.34344 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:34:44.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:43 smithi148 ceph-mon[49387]: osdmap e117: 8 total, 8 up, 8 in 2024-09-21T12:34:44.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:43 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/1981107811' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:34:44.827 INFO:teuthology.orchestra.run.smithi078.stdout: 5 218317 9.68 MB/sec execute 442 sec latency 929.096 ms 2024-09-21T12:34:44.907 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:44 smithi078 ceph-mon[63866]: pgmap v190: 97 pgs: 3 peering, 6 active+recovery_wait+undersized+degraded+remapped, 7 activating+undersized+degraded+remapped, 81 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.0 MiB/s wr, 311 op/s; 378/1671 objects degraded (22.621%); 43 KiB/s, 0 objects/s recovering 2024-09-21T12:34:44.907 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:44 smithi078 ceph-mon[63866]: osdmap e118: 8 total, 8 up, 8 in 2024-09-21T12:34:45.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:44 smithi148 ceph-mon[49387]: pgmap v190: 97 pgs: 3 peering, 6 active+recovery_wait+undersized+degraded+remapped, 7 activating+undersized+degraded+remapped, 81 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.0 MiB/s wr, 311 op/s; 378/1671 objects degraded (22.621%); 43 KiB/s, 0 objects/s recovering 2024-09-21T12:34:45.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:44 smithi148 ceph-mon[49387]: osdmap e118: 8 total, 8 up, 8 in 2024-09-21T12:34:45.660 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:45 smithi078 ceph-mon[63866]: osdmap e119: 8 total, 8 up, 8 in 2024-09-21T12:34:45.660 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:45.660 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:34:45.828 INFO:teuthology.orchestra.run.smithi078.stdout: 5 218794 9.68 MB/sec execute 443 sec latency 45.366 ms 2024-09-21T12:34:46.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:45 smithi148 ceph-mon[49387]: osdmap e119: 8 total, 8 up, 8 in 2024-09-21T12:34:46.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:46.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:34:46.682 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:46 smithi078 ceph-mon[63866]: pgmap v193: 97 pgs: 3 peering, 6 active+recovery_wait+undersized+degraded+remapped, 6 activating+undersized+degraded+remapped, 82 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.2 MiB/s wr, 388 op/s; 351/1638 objects degraded (21.429%); 45 KiB/s, 7 objects/s recovering 2024-09-21T12:34:46.828 INFO:teuthology.orchestra.run.smithi078.stdout: 5 219253 9.68 MB/sec execute 444 sec latency 37.365 ms 2024-09-21T12:34:47.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:46 smithi148 ceph-mon[49387]: pgmap v193: 97 pgs: 3 peering, 6 active+recovery_wait+undersized+degraded+remapped, 6 activating+undersized+degraded+remapped, 82 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.2 MiB/s wr, 388 op/s; 351/1638 objects degraded (21.429%); 45 KiB/s, 7 objects/s recovering 2024-09-21T12:34:47.685 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:47 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 351/1638 objects degraded (21.429%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T12:34:47.828 INFO:teuthology.orchestra.run.smithi078.stdout: 5 219721 9.69 MB/sec execute 445 sec latency 31.722 ms 2024-09-21T12:34:48.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:47 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 351/1638 objects degraded (21.429%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T12:34:48.705 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:48 smithi078 ceph-mon[63866]: pgmap v194: 97 pgs: 97 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.2 MiB/s wr, 360 op/s; 1.2 MiB/s, 70 objects/s recovering 2024-09-21T12:34:48.705 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:48 smithi078 ceph-mon[63866]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 351/1638 objects degraded (21.429%), 12 pgs degraded) 2024-09-21T12:34:48.705 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:48 smithi078 ceph-mon[63866]: Cluster is now healthy 2024-09-21T12:34:48.828 INFO:teuthology.orchestra.run.smithi078.stdout: 5 220012 9.69 MB/sec execute 446 sec latency 46.368 ms 2024-09-21T12:34:49.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:48 smithi148 ceph-mon[49387]: pgmap v194: 97 pgs: 97 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.2 MiB/s wr, 360 op/s; 1.2 MiB/s, 70 objects/s recovering 2024-09-21T12:34:49.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:48 smithi148 ceph-mon[49387]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 351/1638 objects degraded (21.429%), 12 pgs degraded) 2024-09-21T12:34:49.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:48 smithi148 ceph-mon[49387]: Cluster is now healthy 2024-09-21T12:34:49.828 INFO:teuthology.orchestra.run.smithi078.stdout: 5 220371 9.70 MB/sec execute 447 sec latency 50.256 ms 2024-09-21T12:34:50.798 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:50 smithi078 ceph-mon[63866]: pgmap v195: 97 pgs: 97 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 2.7 MiB/s wr, 308 op/s; 1.6 MiB/s, 60 objects/s recovering 2024-09-21T12:34:50.828 INFO:teuthology.orchestra.run.smithi078.stdout: 5 220632 9.71 MB/sec execute 448 sec latency 57.178 ms 2024-09-21T12:34:51.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:50 smithi148 ceph-mon[49387]: pgmap v195: 97 pgs: 97 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 2.7 MiB/s wr, 308 op/s; 1.6 MiB/s, 60 objects/s recovering 2024-09-21T12:34:51.828 INFO:teuthology.orchestra.run.smithi078.stdout: 5 220967 9.71 MB/sec execute 449 sec latency 49.183 ms 2024-09-21T12:34:52.822 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:52 smithi078 ceph-mon[63866]: pgmap v196: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 5.4 MiB/s wr, 559 op/s; 1.4 MiB/s, 52 objects/s recovering 2024-09-21T12:34:52.828 INFO:teuthology.orchestra.run.smithi078.stdout: 5 221384 9.70 MB/sec execute 450 sec latency 33.139 ms 2024-09-21T12:34:53.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:52 smithi148 ceph-mon[49387]: pgmap v196: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 5.4 MiB/s wr, 559 op/s; 1.4 MiB/s, 52 objects/s recovering 2024-09-21T12:34:53.828 INFO:teuthology.orchestra.run.smithi078.stdout: 5 221644 9.68 MB/sec execute 451 sec latency 54.075 ms 2024-09-21T12:34:54.829 INFO:teuthology.orchestra.run.smithi078.stdout: 5 221930 9.68 MB/sec execute 452 sec latency 93.268 ms 2024-09-21T12:34:54.920 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:54 smithi078 ceph-mon[63866]: pgmap v197: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 467 op/s; 1.1 MiB/s, 42 objects/s recovering 2024-09-21T12:34:55.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:54 smithi148 ceph-mon[49387]: pgmap v197: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 467 op/s; 1.1 MiB/s, 42 objects/s recovering 2024-09-21T12:34:55.828 INFO:teuthology.orchestra.run.smithi078.stdout: 5 222321 9.68 MB/sec execute 453 sec latency 73.731 ms 2024-09-21T12:34:55.971 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:55 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:55.971 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:55 smithi078 ceph-mon[63866]: pgmap v198: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.7 MiB/s wr, 444 op/s; 1.0 MiB/s, 38 objects/s recovering 2024-09-21T12:34:56.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:55 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:56.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:55 smithi148 ceph-mon[49387]: pgmap v198: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.7 MiB/s wr, 444 op/s; 1.0 MiB/s, 38 objects/s recovering 2024-09-21T12:34:56.828 INFO:teuthology.orchestra.run.smithi078.stdout: 5 222690 9.67 MB/sec execute 454 sec latency 47.353 ms 2024-09-21T12:34:56.993 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:56 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-21T12:34:56.993 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:56 smithi078 ceph-mon[63866]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T12:34:56.993 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:56 smithi078 ceph-mon[63866]: Upgrade: osd.4 is safe to restart 2024-09-21T12:34:57.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:56 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-21T12:34:57.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:56 smithi148 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T12:34:57.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:56 smithi148 ceph-mon[49387]: Upgrade: osd.4 is safe to restart 2024-09-21T12:34:57.828 INFO:teuthology.orchestra.run.smithi078.stdout: 5 223080 9.67 MB/sec execute 455 sec latency 40.150 ms 2024-09-21T12:34:58.009 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:57 smithi078 ceph-mon[63866]: pgmap v199: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 5.1 MiB/s wr, 489 op/s; 952 KiB/s, 32 objects/s recovering 2024-09-21T12:34:58.010 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:57 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:58.010 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:57 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T12:34:58.010 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:57 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:34:58.277 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:57 smithi148 ceph-mon[49387]: pgmap v199: 97 pgs: 97 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 5.1 MiB/s wr, 489 op/s; 952 KiB/s, 32 objects/s recovering 2024-09-21T12:34:58.278 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:57 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:34:58.278 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:57 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T12:34:58.278 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:57 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:34:58.828 INFO:teuthology.orchestra.run.smithi078.stdout: 5 223440 9.68 MB/sec execute 456 sec latency 50.688 ms 2024-09-21T12:34:59.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:58 smithi078 ceph-mon[63866]: Upgrade: Updating osd.4 2024-09-21T12:34:59.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:58 smithi078 ceph-mon[63866]: Deploying daemon osd.4 on smithi148 2024-09-21T12:34:59.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:58 smithi148 ceph-mon[49387]: Upgrade: Updating osd.4 2024-09-21T12:34:59.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:58 smithi148 ceph-mon[49387]: Deploying daemon osd.4 on smithi148 2024-09-21T12:34:59.828 INFO:teuthology.orchestra.run.smithi078.stdout: 5 223798 9.69 MB/sec execute 457 sec latency 68.996 ms 2024-09-21T12:35:00.089 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:59 smithi078 ceph-mon[63866]: pgmap v200: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 4.3 MiB/s wr, 435 op/s 2024-09-21T12:35:00.089 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:59 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:00.089 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:34:59 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:35:00.186 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:59 smithi148 ceph-mon[49387]: pgmap v200: 97 pgs: 97 active+clean; 334 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 4.3 MiB/s wr, 435 op/s 2024-09-21T12:35:00.186 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:59 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:00.186 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:34:59 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:35:00.828 INFO:teuthology.orchestra.run.smithi078.stdout: 5 224107 9.70 MB/sec execute 458 sec latency 94.320 ms 2024-09-21T12:35:01.828 INFO:teuthology.orchestra.run.smithi078.stdout: 5 224340 9.70 MB/sec execute 459 sec latency 84.721 ms 2024-09-21T12:35:01.964 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:01 smithi148 systemd[1]: Stopping Ceph osd.4 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:35:02.300 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:01 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4[30019]: 2024-09-21T12:35:01.961+0000 7f5968e2f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T12:35:02.300 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:01 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4[30019]: 2024-09-21T12:35:01.961+0000 7f5968e2f700 -1 osd.4 119 *** Got signal Terminated *** 2024-09-21T12:35:02.300 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:01 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4[30019]: 2024-09-21T12:35:01.961+0000 7f5968e2f700 -1 osd.4 119 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T12:35:02.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:02 smithi078 ceph-mon[63866]: pgmap v201: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 6.3 MiB/s wr, 628 op/s 2024-09-21T12:35:02.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:02 smithi078 ceph-mon[63866]: osd.4 marked itself down and dead 2024-09-21T12:35:02.828 INFO:teuthology.orchestra.run.smithi078.stdout: 5 224495 9.68 MB/sec execute 460 sec latency 240.494 ms 2024-09-21T12:35:03.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:02 smithi148 ceph-mon[49387]: pgmap v201: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 6.3 MiB/s wr, 628 op/s 2024-09-21T12:35:03.072 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:02 smithi148 ceph-mon[49387]: osd.4 marked itself down and dead 2024-09-21T12:35:03.473 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:03 smithi148 podman[52805]: 2024-09-21 12:35:03.193841193 +0000 UTC m=+1.379735023 container died 62eefdd572c0450fa15d419e825057f894fbd9e1bd5c718f12523722d4227a77 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4, name=centos-stream, version=8, GIT_BRANCH=HEAD, release=754, RELEASE=HEAD, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, ceph=True, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-09-21T12:35:03.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:03 smithi148 ceph-mon[49387]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T12:35:03.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:03 smithi148 ceph-mon[49387]: osdmap e120: 8 total, 7 up, 8 in 2024-09-21T12:35:03.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:03 smithi148 ceph-mon[49387]: osdmap e121: 8 total, 7 up, 8 in 2024-09-21T12:35:03.801 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:03 smithi148 podman[52805]: 2024-09-21 12:35:03.576691045 +0000 UTC m=+1.762607138 container cleanup 62eefdd572c0450fa15d419e825057f894fbd9e1bd5c718f12523722d4227a77 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, io.openshift.expose-services=, vendor=Red Hat, Inc., version=8, com.redhat.component=centos-stream-container, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, architecture=x86_64, distribution-scope=public, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-21T12:35:03.801 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:03 smithi148 bash[52805]: ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4 2024-09-21T12:35:03.828 INFO:teuthology.orchestra.run.smithi078.stdout: 5 224516 9.67 MB/sec execute 461 sec latency 1110.563 ms 2024-09-21T12:35:03.882 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:03 smithi078 ceph-mon[63866]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T12:35:03.882 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:03 smithi078 ceph-mon[63866]: osdmap e120: 8 total, 7 up, 8 in 2024-09-21T12:35:03.882 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:03 smithi078 ceph-mon[63866]: osdmap e121: 8 total, 7 up, 8 in 2024-09-21T12:35:04.242 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:03 smithi148 podman[52821]: 2024-09-21 12:35:03.881588715 +0000 UTC m=+0.678124402 container remove 62eefdd572c0450fa15d419e825057f894fbd9e1bd5c718f12523722d4227a77 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4, ceph=True, com.redhat.component=centos-stream-container, vcs-type=git, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, name=centos-stream, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, RELEASE=HEAD, vendor=Red Hat, Inc., release=754, GIT_CLEAN=True) 2024-09-21T12:35:04.550 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:04 smithi148 podman[52908]: 2024-09-21 12:35:04.144338197 +0000 UTC m=+0.018621850 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:35:04.551 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:04 smithi148 podman[52908]: 2024-09-21 12:35:04.282577574 +0000 UTC m=+0.156861215 container create 35a1284bd0240ec34b0629bb405f96bbfd87d2882fdfe9b79834b600a5210d1b (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-21T12:35:04.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:04 smithi078 ceph-mon[63866]: pgmap v203: 97 pgs: 2 peering, 12 stale+active+clean, 83 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.2 MiB/s wr, 519 op/s 2024-09-21T12:35:04.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:04 smithi078 ceph-mon[63866]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T12:35:04.828 INFO:teuthology.orchestra.run.smithi078.stdout: 5 224716 9.65 MB/sec execute 462 sec latency 56.986 ms 2024-09-21T12:35:05.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:04 smithi148 ceph-mon[49387]: pgmap v203: 97 pgs: 2 peering, 12 stale+active+clean, 83 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.2 MiB/s wr, 519 op/s 2024-09-21T12:35:05.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:04 smithi148 ceph-mon[49387]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T12:35:05.051 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:04 smithi148 podman[52908]: 2024-09-21 12:35:04.572265665 +0000 UTC m=+0.446549313 container init 35a1284bd0240ec34b0629bb405f96bbfd87d2882fdfe9b79834b600a5210d1b (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T12:35:05.051 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:04 smithi148 podman[52908]: 2024-09-21 12:35:04.576823089 +0000 UTC m=+0.451106730 container start 35a1284bd0240ec34b0629bb405f96bbfd87d2882fdfe9b79834b600a5210d1b (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T12:35:05.051 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:04 smithi148 podman[52908]: 2024-09-21 12:35:04.710948217 +0000 UTC m=+0.585231858 container attach 35a1284bd0240ec34b0629bb405f96bbfd87d2882fdfe9b79834b600a5210d1b (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4-deactivate, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-21T12:35:05.051 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:04 smithi148 podman[52908]: 2024-09-21 12:35:04.789926093 +0000 UTC m=+0.664209744 container died 35a1284bd0240ec34b0629bb405f96bbfd87d2882fdfe9b79834b600a5210d1b (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-21T12:35:05.828 INFO:teuthology.orchestra.run.smithi078.stdout: 5 224898 9.63 MB/sec execute 463 sec latency 65.741 ms 2024-09-21T12:35:05.925 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:05 smithi078 ceph-mon[63866]: Health check failed: Degraded data redundancy: 101/1668 objects degraded (6.055%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T12:35:06.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:05 smithi148 ceph-mon[49387]: Health check failed: Degraded data redundancy: 101/1668 objects degraded (6.055%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T12:35:06.051 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:05 smithi148 podman[52930]: 2024-09-21 12:35:05.825579891 +0000 UTC m=+1.029626964 container remove 35a1284bd0240ec34b0629bb405f96bbfd87d2882fdfe9b79834b600a5210d1b (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-21-0842-4417849) 2024-09-21T12:35:06.051 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:05 smithi148 systemd[1]: ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.4.service: Deactivated successfully. 2024-09-21T12:35:06.051 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:05 smithi148 systemd[1]: Stopped Ceph osd.4 for 6263deca-7813-11ef-baf6-efdc52797490. 2024-09-21T12:35:06.051 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:05 smithi148 systemd[1]: ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.4.service: Consumed 3min 55.108s CPU time. 2024-09-21T12:35:06.460 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:06 smithi148 systemd[1]: Starting Ceph osd.4 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:35:06.461 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:06 smithi148 podman[53030]: 2024-09-21 12:35:06.215878149 +0000 UTC m=+0.017195659 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:35:06.461 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:06 smithi148 podman[53030]: 2024-09-21 12:35:06.335428918 +0000 UTC m=+0.136746411 container create 22162faff87d68fad8aa9819382bad1c972757226056c43286190bf33422cdb4 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4-activate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T12:35:06.682 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:06 smithi078 ceph-mon[63866]: pgmap v205: 97 pgs: 2 peering, 11 stale+active+clean, 8 active+undersized+degraded, 76 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 3.9 MiB/s wr, 328 op/s; 101/1668 objects degraded (6.055%) 2024-09-21T12:35:06.771 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:06 smithi148 ceph-mon[49387]: pgmap v205: 97 pgs: 2 peering, 11 stale+active+clean, 8 active+undersized+degraded, 76 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 3.9 MiB/s wr, 328 op/s; 101/1668 objects degraded (6.055%) 2024-09-21T12:35:06.771 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:06 smithi148 podman[53030]: 2024-09-21 12:35:06.604947482 +0000 UTC m=+0.406264993 container init 22162faff87d68fad8aa9819382bad1c972757226056c43286190bf33422cdb4 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4-activate, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0) 2024-09-21T12:35:06.771 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:06 smithi148 podman[53030]: 2024-09-21 12:35:06.610088608 +0000 UTC m=+0.411406102 container start 22162faff87d68fad8aa9819382bad1c972757226056c43286190bf33422cdb4 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4-activate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS) 2024-09-21T12:35:06.828 INFO:teuthology.orchestra.run.smithi078.stdout: 5 225161 9.62 MB/sec execute 464 sec latency 90.202 ms 2024-09-21T12:35:07.050 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:06 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4-activate[53044]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:35:07.050 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:06 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4-activate[53044]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:35:07.051 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:06 smithi148 bash[53030]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:35:07.051 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:06 smithi148 bash[53030]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:35:07.051 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:06 smithi148 podman[53030]: 2024-09-21 12:35:06.91374828 +0000 UTC m=+0.715065780 container attach 22162faff87d68fad8aa9819382bad1c972757226056c43286190bf33422cdb4 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4-activate, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-21T12:35:07.828 INFO:teuthology.orchestra.run.smithi078.stdout: 5 225453 9.62 MB/sec execute 465 sec latency 48.863 ms 2024-09-21T12:35:08.797 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:08 smithi078 ceph-mon[63866]: pgmap v206: 97 pgs: 12 active+undersized, 2 peering, 27 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 5.4 MiB/s wr, 505 op/s; 253/1665 objects degraded (15.195%) 2024-09-21T12:35:08.828 INFO:teuthology.orchestra.run.smithi078.stdout: 5 225783 9.61 MB/sec execute 466 sec latency 54.304 ms 2024-09-21T12:35:09.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:08 smithi148 ceph-mon[49387]: pgmap v206: 97 pgs: 12 active+undersized, 2 peering, 27 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 5.4 MiB/s wr, 505 op/s; 253/1665 objects degraded (15.195%) 2024-09-21T12:35:09.824 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:09 smithi078 ceph-mon[63866]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T12:35:09.828 INFO:teuthology.orchestra.run.smithi078.stdout: 5 226200 9.61 MB/sec execute 467 sec latency 42.300 ms 2024-09-21T12:35:10.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:09 smithi148 ceph-mon[49387]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T12:35:10.051 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:09 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4-activate[53044]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T12:35:10.051 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:09 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4-activate[53044]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:35:10.051 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:09 smithi148 bash[53030]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T12:35:10.051 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:09 smithi148 bash[53030]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:35:10.051 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:09 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4-activate[53044]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:35:10.051 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:09 smithi148 bash[53030]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:35:10.051 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:09 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4-activate[53044]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T12:35:10.051 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:09 smithi148 bash[53030]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T12:35:10.052 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:09 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4-activate[53044]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-4 --no-mon-config 2024-09-21T12:35:10.052 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:09 smithi148 bash[53030]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-4 --no-mon-config 2024-09-21T12:35:10.469 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:10 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4-activate[53044]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-21T12:35:10.469 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:10 smithi148 bash[53030]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-21T12:35:10.469 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:10 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4-activate[53044]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-4/block 2024-09-21T12:35:10.469 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:10 smithi148 bash[53030]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-4/block 2024-09-21T12:35:10.469 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:10 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4-activate[53044]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T12:35:10.469 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:10 smithi148 bash[53030]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T12:35:10.469 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:10 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4-activate[53044]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T12:35:10.470 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:10 smithi148 bash[53030]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T12:35:10.470 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:10 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4-activate[53044]: --> ceph-volume lvm activate successful for osd ID: 4 2024-09-21T12:35:10.470 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:10 smithi148 bash[53030]: --> ceph-volume lvm activate successful for osd ID: 4 2024-09-21T12:35:10.470 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:10 smithi148 podman[53030]: 2024-09-21 12:35:10.103554409 +0000 UTC m=+3.904871901 container died 22162faff87d68fad8aa9819382bad1c972757226056c43286190bf33422cdb4 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4-activate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux ) 2024-09-21T12:35:10.828 INFO:teuthology.orchestra.run.smithi078.stdout: 5 226511 9.60 MB/sec execute 468 sec latency 48.059 ms 2024-09-21T12:35:10.838 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:10 smithi078 ceph-mon[63866]: pgmap v207: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 2.4 MiB/s wr, 218 op/s; 264/1656 objects degraded (15.942%) 2024-09-21T12:35:11.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:10 smithi148 ceph-mon[49387]: pgmap v207: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 2.4 MiB/s wr, 218 op/s; 264/1656 objects degraded (15.942%) 2024-09-21T12:35:11.050 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:10 smithi148 podman[53247]: 2024-09-21 12:35:10.969935307 +0000 UTC m=+0.859659677 container remove 22162faff87d68fad8aa9819382bad1c972757226056c43286190bf33422cdb4 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4-activate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-21-0842-4417849, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=) 2024-09-21T12:35:11.475 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:11 smithi148 podman[53303]: 2024-09-21 12:35:11.083385943 +0000 UTC m=+0.017394508 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:35:11.476 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:11 smithi148 podman[53303]: 2024-09-21 12:35:11.220993615 +0000 UTC m=+0.155002182 container create 10ec7d6b476e5f74d7548f339428e9fe901dbaea30143793e4ce97383d924d5b (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0842-4417849) 2024-09-21T12:35:11.476 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:11 smithi148 podman[53303]: 2024-09-21 12:35:11.474864906 +0000 UTC m=+0.408873471 container init 10ec7d6b476e5f74d7548f339428e9fe901dbaea30143793e4ce97383d924d5b (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T12:35:11.800 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:11 smithi148 podman[53303]: 2024-09-21 12:35:11.479325432 +0000 UTC m=+0.413333997 container start 10ec7d6b476e5f74d7548f339428e9fe901dbaea30143793e4ce97383d924d5b (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-21T12:35:11.800 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:11 smithi148 ceph-osd[53337]: -- 172.21.15.148:0/3477379207 <== mon.1 v2:172.21.15.148:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5590f2a0c1e0 con 0x5590f29a0000 2024-09-21T12:35:11.801 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:11 smithi148 bash[53303]: 10ec7d6b476e5f74d7548f339428e9fe901dbaea30143793e4ce97383d924d5b 2024-09-21T12:35:11.801 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:11 smithi148 systemd[1]: Started Ceph osd.4 for 6263deca-7813-11ef-baf6-efdc52797490. 2024-09-21T12:35:11.828 INFO:teuthology.orchestra.run.smithi078.stdout: 5 226918 9.61 MB/sec execute 469 sec latency 43.188 ms 2024-09-21T12:35:12.828 INFO:teuthology.orchestra.run.smithi078.stdout: 5 227296 9.62 MB/sec execute 470 sec latency 42.781 ms 2024-09-21T12:35:12.873 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:12 smithi078 ceph-mon[63866]: pgmap v208: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 328 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 2.8 MiB/s wr, 261 op/s; 246/1548 objects degraded (15.891%) 2024-09-21T12:35:12.874 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:12 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:12.874 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:12 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:12.874 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:12 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:35:12.874 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:12 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 246/1548 objects degraded (15.891%), 28 pgs degraded (PG_DEGRADED) 2024-09-21T12:35:13.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:12 smithi148 ceph-mon[49387]: pgmap v208: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 328 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 2.8 MiB/s wr, 261 op/s; 246/1548 objects degraded (15.891%) 2024-09-21T12:35:13.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:12 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:13.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:12 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:13.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:12 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:35:13.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:12 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 246/1548 objects degraded (15.891%), 28 pgs degraded (PG_DEGRADED) 2024-09-21T12:35:13.402 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:13 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4[53317]: 2024-09-21T12:35:13.361+0000 7f7d7f98d640 -1 Falling back to public interface 2024-09-21T12:35:13.829 INFO:teuthology.orchestra.run.smithi078.stdout: 5 227618 9.63 MB/sec execute 471 sec latency 44.148 ms 2024-09-21T12:35:13.951 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-21T12:35:14.678 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:14 smithi078 ceph-mon[63866]: pgmap v209: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.7 MiB/s wr, 449 op/s; 251/1587 objects degraded (15.816%) 2024-09-21T12:35:14.829 INFO:teuthology.orchestra.run.smithi078.stdout: 5 227902 9.63 MB/sec execute 472 sec latency 50.731 ms 2024-09-21T12:35:14.851 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T12:35:14.852 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (7m) 40s ago 16m 20.0M - 0.27.0 11f11916f8cd e6e67ab07dec 2024-09-21T12:35:14.852 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (4m) 40s ago 16m 6895k - 19.3.0-5104-g44178494 b415816db8b4 46a10ea0b111 2024-09-21T12:35:14.852 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi148 smithi148 running (4m) 3m ago 15m 6895k - 19.3.0-5104-g44178494 b415816db8b4 23deb0615621 2024-09-21T12:35:14.852 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (7m) 40s ago 16m 53.3M - dad864ee21e9 2a140c36091b 2024-09-21T12:35:14.852 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.ytpzwh smithi078 running (10m) 40s ago 10m 38.4M - 17.2.0 e1d6a67b021e 5e9b109502a2 2024-09-21T12:35:14.852 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi148.uukezr smithi148 running (10m) 3m ago 10m 13.7M - 17.2.0 e1d6a67b021e b7e8aa0f3a6c 2024-09-21T12:35:14.852 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.ntwyan smithi078 *:8443,9283,8765 running (6m) 40s ago 17m 519M - 19.3.0-5104-g44178494 b415816db8b4 a61baedebf43 2024-09-21T12:35:14.852 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi148.oyiqmx smithi148 *:8443,9283,8765 running (5m) 3m ago 15m 454M - 19.3.0-5104-g44178494 b415816db8b4 0255535c6fdb 2024-09-21T12:35:14.852 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (5m) 40s ago 17m 55.9M 2048M 19.3.0-5104-g44178494 b415816db8b4 22b3097efbd8 2024-09-21T12:35:14.852 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi148 smithi148 running (4m) 3m ago 15m 36.1M 2048M 19.3.0-5104-g44178494 b415816db8b4 df8b08509136 2024-09-21T12:35:14.852 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.ydbhru smithi078 *:2049 running (10m) 40s ago 10m 279M - 4.0 e1d6a67b021e 7e5b0274ac38 2024-09-21T12:35:14.853 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi148.einuzt smithi148 *:2049 running (9m) 3m ago 9m 86.8M - 4.0 e1d6a67b021e 76a8b0eabede 2024-09-21T12:35:14.853 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (7m) 40s ago 16m 11.0M - 1.7.0 72c9c2088986 301adb4e9e22 2024-09-21T12:35:14.853 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi148 smithi148 *:9100 running (6m) 3m ago 15m 11.3M - 1.7.0 72c9c2088986 0cceff4bf342 2024-09-21T12:35:14.853 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (3m) 40s ago 14m 345M 1936M 19.3.0-5104-g44178494 b415816db8b4 7374add97534 2024-09-21T12:35:14.853 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (2m) 40s ago 14m 276M 1936M 19.3.0-5104-g44178494 b415816db8b4 14655a8da4e5 2024-09-21T12:35:14.853 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (104s) 40s ago 13m 219M 1936M 19.3.0-5104-g44178494 b415816db8b4 07ceda32f383 2024-09-21T12:35:14.853 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (51s) 40s ago 13m 11.4M 1936M 19.3.0-5104-g44178494 b415816db8b4 ead87e4306ff 2024-09-21T12:35:14.853 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi148 starting - - - 2704M 2024-09-21T12:35:14.853 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi148 running (12m) 3m ago 12m 314M 2704M 17.2.0 e1d6a67b021e 7b44be78baca 2024-09-21T12:35:14.853 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi148 running (12m) 3m ago 12m 509M 2704M 17.2.0 e1d6a67b021e 3d23057910ea 2024-09-21T12:35:14.853 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi148 running (11m) 3m ago 11m 334M 2704M 17.2.0 e1d6a67b021e f9e4db9d35fa 2024-09-21T12:35:14.853 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (5m) 40s ago 15m 55.8M - 2.51.0 1d3b7f56885b face2d4cdd1b 2024-09-21T12:35:14.945 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:14 smithi148 ceph-mon[49387]: pgmap v209: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.7 MiB/s wr, 449 op/s; 251/1587 objects degraded (15.816%) 2024-09-21T12:35:15.403 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:35:15.403 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-21T12:35:15.403 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:35:15.403 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:35:15.403 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-21T12:35:15.403 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:35:15.403 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:35:15.403 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-21T12:35:15.403 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-21T12:35:15.403 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 4 2024-09-21T12:35:15.404 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:35:15.404 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-21T12:35:15.404 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:35:15.404 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:35:15.404 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-21T12:35:15.404 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-21T12:35:15.404 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 8 2024-09-21T12:35:15.404 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T12:35:15.404 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:35:15.710 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:15 smithi078 ceph-mon[63866]: from='client.44263 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:35:15.710 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:15 smithi078 ceph-mon[63866]: from='client.34356 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:35:15.710 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:15 smithi078 ceph-mon[63866]: from='client.34360 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:35:15.710 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:15 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:15.710 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:15 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:35:15.710 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:15 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:15.710 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:15 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:15.710 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:15 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/2821003963' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:35:15.829 INFO:teuthology.orchestra.run.smithi078.stdout: 5 228141 9.62 MB/sec execute 473 sec latency 75.405 ms 2024-09-21T12:35:15.893 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:35:15.893 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5", 2024-09-21T12:35:15.893 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-21T12:35:15.894 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T12:35:15.894 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-21T12:35:15.894 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-21T12:35:15.894 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-09-21T12:35:15.894 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr" 2024-09-21T12:35:15.894 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-21T12:35:15.894 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "11/23 daemons upgraded", 2024-09-21T12:35:15.894 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T12:35:15.894 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-21T12:35:15.894 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:35:16.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:15 smithi148 ceph-mon[49387]: from='client.44263 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:35:16.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:15 smithi148 ceph-mon[49387]: from='client.34356 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:35:16.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:15 smithi148 ceph-mon[49387]: from='client.34360 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:35:16.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:15 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:16.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:15 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:35:16.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:15 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:16.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:15 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:16.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:15 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/2821003963' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:35:16.460 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 260/1614 objects degraded (16.109%), 28 pgs degraded 2024-09-21T12:35:16.460 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-21T12:35:16.460 INFO:teuthology.orchestra.run.smithi078.stdout: osd.4 (root=default,host=smithi148) is down 2024-09-21T12:35:16.460 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 260/1614 objects degraded (16.109%), 28 pgs degraded 2024-09-21T12:35:16.460 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.5 is active+undersized+degraded, acting [7,0] 2024-09-21T12:35:16.460 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.6 is active+undersized+degraded, acting [1,6] 2024-09-21T12:35:16.460 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.b is active+undersized+degraded, acting [7,5] 2024-09-21T12:35:16.460 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.d is active+undersized+degraded, acting [1,3] 2024-09-21T12:35:16.461 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.f is active+undersized+degraded, acting [0,7] 2024-09-21T12:35:16.461 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.11 is active+undersized+degraded, acting [6,1] 2024-09-21T12:35:16.461 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.13 is active+undersized+degraded, acting [0,3] 2024-09-21T12:35:16.461 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.18 is active+undersized+degraded, acting [5,2] 2024-09-21T12:35:16.461 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1a is active+undersized+degraded, acting [6,7] 2024-09-21T12:35:16.461 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1f is active+undersized+degraded, acting [0,7] 2024-09-21T12:35:16.461 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1 is active+undersized+degraded, acting [0,3] 2024-09-21T12:35:16.461 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.3 is active+undersized+degraded, acting [0,6] 2024-09-21T12:35:16.461 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.6 is active+undersized+degraded, acting [0,1] 2024-09-21T12:35:16.461 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.9 is active+undersized+degraded, acting [2,7] 2024-09-21T12:35:16.461 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.a is active+undersized+degraded, acting [6,1] 2024-09-21T12:35:16.461 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.b is active+undersized+degraded, acting [3,0] 2024-09-21T12:35:16.461 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.e is active+undersized+degraded, acting [7,1] 2024-09-21T12:35:16.461 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.f is active+undersized+degraded, acting [7,0] 2024-09-21T12:35:16.461 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.11 is active+undersized+degraded, acting [7,6] 2024-09-21T12:35:16.462 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.13 is active+undersized+degraded, acting [7,2] 2024-09-21T12:35:16.462 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.14 is active+undersized+degraded, acting [7,6] 2024-09-21T12:35:16.462 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.15 is active+undersized+degraded, acting [7,3] 2024-09-21T12:35:16.462 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.19 is active+undersized+degraded, acting [1,3] 2024-09-21T12:35:16.462 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1a is active+undersized+degraded, acting [1,2] 2024-09-21T12:35:16.462 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1b is active+undersized+degraded, acting [0,7] 2024-09-21T12:35:16.462 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1c is active+undersized+degraded, acting [5,1] 2024-09-21T12:35:16.462 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1d is active+undersized+degraded, acting [5,6] 2024-09-21T12:35:16.462 INFO:teuthology.orchestra.run.smithi078.stdout: pg 4.5 is active+undersized+degraded, acting [6,7] 2024-09-21T12:35:16.748 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:16 smithi078 ceph-mon[63866]: pgmap v210: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.6 MiB/s wr, 391 op/s; 260/1614 objects degraded (16.109%) 2024-09-21T12:35:16.748 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:16 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:16.748 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:16 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:16.748 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:16 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/1753895003' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:35:16.751 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:16 smithi148 ceph-mon[49387]: pgmap v210: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.6 MiB/s wr, 391 op/s; 260/1614 objects degraded (16.109%) 2024-09-21T12:35:16.751 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:16 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:16.751 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:16 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:16.751 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:16 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/1753895003' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:35:16.829 INFO:teuthology.orchestra.run.smithi078.stdout: 5 228400 9.61 MB/sec execute 474 sec latency 44.872 ms 2024-09-21T12:35:17.755 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:17 smithi078 ceph-mon[63866]: from='client.34368 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:35:17.756 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:17 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 260/1614 objects degraded (16.109%), 28 pgs degraded (PG_DEGRADED) 2024-09-21T12:35:17.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:17 smithi148 ceph-mon[49387]: from='client.34368 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:35:17.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:17 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 260/1614 objects degraded (16.109%), 28 pgs degraded (PG_DEGRADED) 2024-09-21T12:35:17.829 INFO:teuthology.orchestra.run.smithi078.stdout: 5 228609 9.60 MB/sec execute 475 sec latency 75.769 ms 2024-09-21T12:35:18.784 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:18 smithi078 ceph-mon[63866]: pgmap v211: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.8 MiB/s wr, 474 op/s; 267/1692 objects degraded (15.780%) 2024-09-21T12:35:18.829 INFO:teuthology.orchestra.run.smithi078.stdout: 5 228878 9.59 MB/sec execute 476 sec latency 47.890 ms 2024-09-21T12:35:19.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:18 smithi148 ceph-mon[49387]: pgmap v211: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.8 MiB/s wr, 474 op/s; 267/1692 objects degraded (15.780%) 2024-09-21T12:35:19.829 INFO:teuthology.orchestra.run.smithi078.stdout: 5 229162 9.58 MB/sec execute 477 sec latency 61.672 ms 2024-09-21T12:35:20.550 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:20 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4[53317]: 2024-09-21T12:35:20.194+0000 7f7d7f98d640 -1 osd.4 0 read_superblock omap replica is missing. 2024-09-21T12:35:20.829 INFO:teuthology.orchestra.run.smithi078.stdout: 5 229455 9.57 MB/sec execute 478 sec latency 62.296 ms 2024-09-21T12:35:21.081 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:20 smithi078 ceph-mon[63866]: pgmap v212: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 3.8 MiB/s wr, 369 op/s; 268/1695 objects degraded (15.811%) 2024-09-21T12:35:21.081 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:20 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:21.081 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:20 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:21.081 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:20 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:35:21.081 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:20 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:35:21.081 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:20 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:21.081 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:20 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:21.081 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:20 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:35:21.081 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:20 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:35:21.082 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:20 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:35:21.082 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:20 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:35:21.082 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:20 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-21T12:35:21.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:20 smithi148 ceph-mon[49387]: pgmap v212: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 3.8 MiB/s wr, 369 op/s; 268/1695 objects degraded (15.811%) 2024-09-21T12:35:21.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:20 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:21.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:20 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:21.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:20 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:35:21.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:20 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:35:21.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:20 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:21.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:20 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:21.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:20 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:35:21.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:20 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:35:21.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:20 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:35:21.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:20 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:35:21.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:20 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-21T12:35:21.800 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:21 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4[53317]: 2024-09-21T12:35:21.459+0000 7f7d7f98d640 -1 osd.4 119 log_to_monitors true 2024-09-21T12:35:21.829 INFO:teuthology.orchestra.run.smithi078.stdout: 5 229727 9.56 MB/sec execute 479 sec latency 63.775 ms 2024-09-21T12:35:22.124 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:21 smithi078 ceph-mon[63866]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T12:35:22.125 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:21 smithi078 ceph-mon[63866]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-21T12:35:22.125 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:21 smithi078 ceph-mon[63866]: from='osd.4 [v2:172.21.15.148:6800/613690352,v1:172.21.15.148:6801/613690352]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T12:35:22.125 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:21 smithi078 ceph-mon[63866]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T12:35:22.300 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:35:21 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4[53317]: 2024-09-21T12:35:21.906+0000 7f7d77737640 -1 osd.4 119 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T12:35:22.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:21 smithi148 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T12:35:22.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:21 smithi148 ceph-mon[49387]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-21T12:35:22.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:21 smithi148 ceph-mon[49387]: from='osd.4 [v2:172.21.15.148:6800/613690352,v1:172.21.15.148:6801/613690352]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T12:35:22.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:21 smithi148 ceph-mon[49387]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T12:35:22.829 INFO:teuthology.orchestra.run.smithi078.stdout: 5 229980 9.55 MB/sec execute 480 sec latency 56.968 ms 2024-09-21T12:35:23.157 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:22 smithi078 ceph-mon[63866]: pgmap v213: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 5.0 MiB/s wr, 528 op/s; 240/1569 objects degraded (15.296%) 2024-09-21T12:35:23.158 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:22 smithi078 ceph-mon[63866]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T12:35:23.158 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:22 smithi078 ceph-mon[63866]: from='osd.4 [v2:172.21.15.148:6800/613690352,v1:172.21.15.148:6801/613690352]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi148", "root=default"]} : dispatch 2024-09-21T12:35:23.158 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:22 smithi078 ceph-mon[63866]: osdmap e122: 8 total, 7 up, 8 in 2024-09-21T12:35:23.158 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:22 smithi078 ceph-mon[63866]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi148", "root=default"]} : dispatch 2024-09-21T12:35:23.158 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:22 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 240/1569 objects degraded (15.296%), 28 pgs degraded (PG_DEGRADED) 2024-09-21T12:35:23.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:22 smithi148 ceph-mon[49387]: pgmap v213: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 5.0 MiB/s wr, 528 op/s; 240/1569 objects degraded (15.296%) 2024-09-21T12:35:23.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:22 smithi148 ceph-mon[49387]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T12:35:23.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:22 smithi148 ceph-mon[49387]: from='osd.4 [v2:172.21.15.148:6800/613690352,v1:172.21.15.148:6801/613690352]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi148", "root=default"]} : dispatch 2024-09-21T12:35:23.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:22 smithi148 ceph-mon[49387]: osdmap e122: 8 total, 7 up, 8 in 2024-09-21T12:35:23.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:22 smithi148 ceph-mon[49387]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi148", "root=default"]} : dispatch 2024-09-21T12:35:23.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:22 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 240/1569 objects degraded (15.296%), 28 pgs degraded (PG_DEGRADED) 2024-09-21T12:35:23.829 INFO:teuthology.orchestra.run.smithi078.stdout: 5 230006 9.54 MB/sec execute 481 sec latency 923.887 ms 2024-09-21T12:35:24.160 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:23 smithi078 ceph-mon[63866]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T12:35:24.161 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:23 smithi078 ceph-mon[63866]: osd.4 [v2:172.21.15.148:6800/613690352,v1:172.21.15.148:6801/613690352] boot 2024-09-21T12:35:24.161 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:23 smithi078 ceph-mon[63866]: osdmap e123: 8 total, 8 up, 8 in 2024-09-21T12:35:24.161 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:23 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T12:35:24.161 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:23 smithi078 ceph-mon[63866]: pgmap v216: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.4 MiB/s wr, 429 op/s; 238/1530 objects degraded (15.556%) 2024-09-21T12:35:24.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:23 smithi148 ceph-mon[49387]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T12:35:24.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:23 smithi148 ceph-mon[49387]: osd.4 [v2:172.21.15.148:6800/613690352,v1:172.21.15.148:6801/613690352] boot 2024-09-21T12:35:24.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:23 smithi148 ceph-mon[49387]: osdmap e123: 8 total, 8 up, 8 in 2024-09-21T12:35:24.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:23 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T12:35:24.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:23 smithi148 ceph-mon[49387]: pgmap v216: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.4 MiB/s wr, 429 op/s; 238/1530 objects degraded (15.556%) 2024-09-21T12:35:24.829 INFO:teuthology.orchestra.run.smithi078.stdout: 5 230089 9.52 MB/sec execute 482 sec latency 1923.922 ms 2024-09-21T12:35:25.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:24 smithi078 ceph-mon[63866]: osdmap e124: 8 total, 8 up, 8 in 2024-09-21T12:35:25.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:24 smithi148 ceph-mon[49387]: osdmap e124: 8 total, 8 up, 8 in 2024-09-21T12:35:25.829 INFO:teuthology.orchestra.run.smithi078.stdout: 5 230301 9.52 MB/sec execute 483 sec latency 2163.561 ms 2024-09-21T12:35:25.922 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:25 smithi078 ceph-mon[63866]: osdmap e125: 8 total, 8 up, 8 in 2024-09-21T12:35:25.923 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:25 smithi078 ceph-mon[63866]: pgmap v219: 97 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized, 21 active+undersized+degraded, 58 active+clean; 330 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 881 KiB/s wr, 75 op/s; 264/1509 objects degraded (17.495%); 0 B/s, 0 objects/s recovering 2024-09-21T12:35:26.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:25 smithi148 ceph-mon[49387]: osdmap e125: 8 total, 8 up, 8 in 2024-09-21T12:35:26.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:25 smithi148 ceph-mon[49387]: pgmap v219: 97 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized, 21 active+undersized+degraded, 58 active+clean; 330 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 881 KiB/s wr, 75 op/s; 264/1509 objects degraded (17.495%); 0 B/s, 0 objects/s recovering 2024-09-21T12:35:26.829 INFO:teuthology.orchestra.run.smithi078.stdout: 5 230587 9.52 MB/sec execute 484 sec latency 69.800 ms 2024-09-21T12:35:26.936 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:26 smithi078 ceph-mon[63866]: osdmap e126: 8 total, 8 up, 8 in 2024-09-21T12:35:27.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:26 smithi148 ceph-mon[49387]: osdmap e126: 8 total, 8 up, 8 in 2024-09-21T12:35:27.829 INFO:teuthology.orchestra.run.smithi078.stdout: 5 230765 9.52 MB/sec execute 485 sec latency 864.293 ms 2024-09-21T12:35:27.969 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:27 smithi078 ceph-mon[63866]: osdmap e127: 8 total, 8 up, 8 in 2024-09-21T12:35:27.970 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:27 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 264/1509 objects degraded (17.495%), 26 pgs degraded (PG_DEGRADED) 2024-09-21T12:35:27.970 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:27 smithi078 ceph-mon[63866]: pgmap v222: 97 pgs: 2 active+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 333 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 3.2 MiB/s wr, 306 op/s; 241/1509 objects degraded (15.971%); 1.7 MiB/s, 57 keys/s, 25 objects/s recovering 2024-09-21T12:35:28.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:27 smithi148 ceph-mon[49387]: osdmap e127: 8 total, 8 up, 8 in 2024-09-21T12:35:28.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:27 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 264/1509 objects degraded (17.495%), 26 pgs degraded (PG_DEGRADED) 2024-09-21T12:35:28.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:27 smithi148 ceph-mon[49387]: pgmap v222: 97 pgs: 2 active+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 333 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 3.2 MiB/s wr, 306 op/s; 241/1509 objects degraded (15.971%); 1.7 MiB/s, 57 keys/s, 25 objects/s recovering 2024-09-21T12:35:28.829 INFO:teuthology.orchestra.run.smithi078.stdout: 5 230982 9.51 MB/sec execute 486 sec latency 997.437 ms 2024-09-21T12:35:29.038 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:28 smithi078 ceph-mon[63866]: osdmap e128: 8 total, 8 up, 8 in 2024-09-21T12:35:29.038 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:28 smithi078 ceph-mon[63866]: osdmap e129: 8 total, 8 up, 8 in 2024-09-21T12:35:29.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:28 smithi148 ceph-mon[49387]: osdmap e128: 8 total, 8 up, 8 in 2024-09-21T12:35:29.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:28 smithi148 ceph-mon[49387]: osdmap e129: 8 total, 8 up, 8 in 2024-09-21T12:35:29.829 INFO:teuthology.orchestra.run.smithi078.stdout: 5 231194 9.51 MB/sec execute 487 sec latency 692.410 ms 2024-09-21T12:35:30.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:29 smithi078 ceph-mon[63866]: pgmap v225: 97 pgs: 2 active+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 333 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 2.9 MiB/s wr, 287 op/s; 241/1509 objects degraded (15.971%); 1.7 MiB/s, 57 keys/s, 24 objects/s recovering 2024-09-21T12:35:30.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:29 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:30.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:29 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:35:30.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:29 smithi078 ceph-mon[63866]: osdmap e130: 8 total, 8 up, 8 in 2024-09-21T12:35:30.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:29 smithi148 ceph-mon[49387]: pgmap v225: 97 pgs: 2 active+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 83 active+clean; 333 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 2.9 MiB/s wr, 287 op/s; 241/1509 objects degraded (15.971%); 1.7 MiB/s, 57 keys/s, 24 objects/s recovering 2024-09-21T12:35:30.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:29 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:30.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:29 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:35:30.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:29 smithi148 ceph-mon[49387]: osdmap e130: 8 total, 8 up, 8 in 2024-09-21T12:35:30.829 INFO:teuthology.orchestra.run.smithi078.stdout: 5 231346 9.51 MB/sec execute 488 sec latency 848.910 ms 2024-09-21T12:35:31.829 INFO:teuthology.orchestra.run.smithi078.stdout: 5 231542 9.49 MB/sec execute 489 sec latency 68.718 ms 2024-09-21T12:35:32.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:31 smithi078 ceph-mon[63866]: osdmap e131: 8 total, 8 up, 8 in 2024-09-21T12:35:32.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:31 smithi078 ceph-mon[63866]: pgmap v228: 97 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.1 MiB/s wr, 317 op/s; 88/1629 objects degraded (5.402%); 86 KiB/s, 41 objects/s recovering 2024-09-21T12:35:32.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:31 smithi148 ceph-mon[49387]: osdmap e131: 8 total, 8 up, 8 in 2024-09-21T12:35:32.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:31 smithi148 ceph-mon[49387]: pgmap v228: 97 pgs: 1 peering, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.1 MiB/s wr, 317 op/s; 88/1629 objects degraded (5.402%); 86 KiB/s, 41 objects/s recovering 2024-09-21T12:35:32.829 INFO:teuthology.orchestra.run.smithi078.stdout: 5 231689 9.48 MB/sec execute 490 sec latency 85.291 ms 2024-09-21T12:35:33.205 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:32 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 88/1629 objects degraded (5.402%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T12:35:33.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:32 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 88/1629 objects degraded (5.402%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T12:35:33.829 INFO:teuthology.orchestra.run.smithi078.stdout: 5 231927 9.47 MB/sec execute 491 sec latency 75.716 ms 2024-09-21T12:35:33.976 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:33 smithi078 ceph-mon[63866]: pgmap v229: 97 pgs: 1 peering, 96 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.3 MiB/s wr, 259 op/s; 266 KiB/s, 45 objects/s recovering 2024-09-21T12:35:34.201 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:33 smithi148 ceph-mon[49387]: pgmap v229: 97 pgs: 1 peering, 96 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.3 MiB/s wr, 259 op/s; 266 KiB/s, 45 objects/s recovering 2024-09-21T12:35:34.829 INFO:teuthology.orchestra.run.smithi078.stdout: 5 232141 9.46 MB/sec execute 492 sec latency 39.976 ms 2024-09-21T12:35:34.990 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:34 smithi078 ceph-mon[63866]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 88/1629 objects degraded (5.402%), 4 pgs degraded) 2024-09-21T12:35:34.991 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:34 smithi078 ceph-mon[63866]: Cluster is now healthy 2024-09-21T12:35:35.215 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:34 smithi148 ceph-mon[49387]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 88/1629 objects degraded (5.402%), 4 pgs degraded) 2024-09-21T12:35:35.215 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:34 smithi148 ceph-mon[49387]: Cluster is now healthy 2024-09-21T12:35:35.829 INFO:teuthology.orchestra.run.smithi078.stdout: 5 232361 9.45 MB/sec execute 493 sec latency 73.212 ms 2024-09-21T12:35:36.019 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:35 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-21T12:35:36.020 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:35 smithi078 ceph-mon[63866]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T12:35:36.020 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:35 smithi078 ceph-mon[63866]: Upgrade: osd.5 is safe to restart 2024-09-21T12:35:36.020 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:35 smithi078 ceph-mon[63866]: pgmap v230: 97 pgs: 97 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 4.2 MiB/s wr, 421 op/s; 226 KiB/s, 38 objects/s recovering 2024-09-21T12:35:36.033 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:35 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-21T12:35:36.033 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:35 smithi148 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T12:35:36.033 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:35 smithi148 ceph-mon[49387]: Upgrade: osd.5 is safe to restart 2024-09-21T12:35:36.033 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:35 smithi148 ceph-mon[49387]: pgmap v230: 97 pgs: 97 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 4.2 MiB/s wr, 421 op/s; 226 KiB/s, 38 objects/s recovering 2024-09-21T12:35:36.829 INFO:teuthology.orchestra.run.smithi078.stdout: 5 232637 9.44 MB/sec execute 494 sec latency 62.505 ms 2024-09-21T12:35:37.829 INFO:teuthology.orchestra.run.smithi078.stdout: 5 233011 9.44 MB/sec execute 495 sec latency 73.015 ms 2024-09-21T12:35:38.319 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:38 smithi078 ceph-mon[63866]: Upgrade: Updating osd.5 2024-09-21T12:35:38.319 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:38 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:38.319 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:38 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T12:35:38.319 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:38 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:35:38.319 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:38 smithi078 ceph-mon[63866]: Deploying daemon osd.5 on smithi148 2024-09-21T12:35:38.319 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:38 smithi078 ceph-mon[63866]: pgmap v231: 97 pgs: 97 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 4.3 MiB/s wr, 394 op/s; 187 KiB/s, 31 objects/s recovering 2024-09-21T12:35:38.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:38 smithi148 ceph-mon[49387]: Upgrade: Updating osd.5 2024-09-21T12:35:38.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:38 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:38.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:38 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T12:35:38.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:38 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:35:38.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:38 smithi148 ceph-mon[49387]: Deploying daemon osd.5 on smithi148 2024-09-21T12:35:38.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:38 smithi148 ceph-mon[49387]: pgmap v231: 97 pgs: 97 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 4.3 MiB/s wr, 394 op/s; 187 KiB/s, 31 objects/s recovering 2024-09-21T12:35:38.829 INFO:teuthology.orchestra.run.smithi078.stdout: 5 233370 9.43 MB/sec execute 496 sec latency 51.063 ms 2024-09-21T12:35:39.830 INFO:teuthology.orchestra.run.smithi078.stdout: 5 233783 9.44 MB/sec execute 497 sec latency 50.942 ms 2024-09-21T12:35:40.550 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:40 smithi148 systemd[1]: Stopping Ceph osd.5 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:35:40.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:40 smithi078 ceph-mon[63866]: pgmap v232: 97 pgs: 97 active+clean; 332 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.7 MiB/s wr, 370 op/s; 156 KiB/s, 26 objects/s recovering 2024-09-21T12:35:40.830 INFO:teuthology.orchestra.run.smithi078.stdout: 5 234171 9.45 MB/sec execute 498 sec latency 49.393 ms 2024-09-21T12:35:41.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:40 smithi148 ceph-mon[49387]: pgmap v232: 97 pgs: 97 active+clean; 332 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.7 MiB/s wr, 370 op/s; 156 KiB/s, 26 objects/s recovering 2024-09-21T12:35:41.051 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:40 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[32971]: 2024-09-21T12:35:40.815+0000 7fb53f29c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T12:35:41.051 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:40 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[32971]: 2024-09-21T12:35:40.815+0000 7fb53f29c700 -1 osd.5 131 *** Got signal Terminated *** 2024-09-21T12:35:41.051 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:40 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[32971]: 2024-09-21T12:35:40.815+0000 7fb53f29c700 -1 osd.5 131 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T12:35:41.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:41 smithi078 ceph-mon[63866]: osd.5 marked itself down and dead 2024-09-21T12:35:41.829 INFO:teuthology.orchestra.run.smithi078.stdout: 5 234496 9.46 MB/sec execute 499 sec latency 238.716 ms 2024-09-21T12:35:42.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:41 smithi148 ceph-mon[49387]: osd.5 marked itself down and dead 2024-09-21T12:35:42.534 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:42 smithi148 podman[54889]: 2024-09-21 12:35:42.244915415 +0000 UTC m=+1.563570432 container died 7b44be78bacac759f575b576ed9ff8720998d4412edaaffe480e2707d6425085 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5, GIT_BRANCH=HEAD, RELEASE=HEAD, GIT_CLEAN=True, ceph=True, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-21T12:35:42.800 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:42 smithi148 podman[54889]: 2024-09-21 12:35:42.632627055 +0000 UTC m=+1.951282078 container cleanup 7b44be78bacac759f575b576ed9ff8720998d4412edaaffe480e2707d6425085 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, architecture=x86_64, io.buildah.version=1.19.8, release=754, io.k8s.display-name=CentOS Stream 8, version=8, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0) 2024-09-21T12:35:42.801 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:42 smithi148 bash[54889]: ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5 2024-09-21T12:35:42.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:42 smithi148 ceph-mon[49387]: pgmap v233: 97 pgs: 97 active+clean; 332 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.4 MiB/s wr, 316 op/s; 141 KiB/s, 11 objects/s recovering 2024-09-21T12:35:42.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:42 smithi148 ceph-mon[49387]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T12:35:42.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:42 smithi148 ceph-mon[49387]: osdmap e132: 8 total, 7 up, 8 in 2024-09-21T12:35:42.830 INFO:teuthology.orchestra.run.smithi078.stdout: 5 234563 9.44 MB/sec execute 500 sec latency 1036.593 ms 2024-09-21T12:35:42.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:42 smithi078 ceph-mon[63866]: pgmap v233: 97 pgs: 97 active+clean; 332 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.4 MiB/s wr, 316 op/s; 141 KiB/s, 11 objects/s recovering 2024-09-21T12:35:42.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:42 smithi078 ceph-mon[63866]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T12:35:42.850 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:42 smithi078 ceph-mon[63866]: osdmap e132: 8 total, 7 up, 8 in 2024-09-21T12:35:43.251 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:42 smithi148 podman[54904]: 2024-09-21 12:35:42.9098854 +0000 UTC m=+0.667143541 container remove 7b44be78bacac759f575b576ed9ff8720998d4412edaaffe480e2707d6425085 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5, version=8, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, RELEASE=HEAD, vcs-type=git, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., distribution-scope=public, io.openshift.expose-services=, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True) 2024-09-21T12:35:43.550 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:43 smithi148 podman[54991]: 2024-09-21 12:35:43.153401567 +0000 UTC m=+0.016428064 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:35:43.550 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:43 smithi148 podman[54991]: 2024-09-21 12:35:43.294032013 +0000 UTC m=+0.157058509 container create 6d7810eb0e7911c7df49bf806eae1780904a5ca29b50b9043aefb3d80db4a7da (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0842-4417849, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T12:35:43.629 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:43 smithi078 ceph-mon[63866]: osdmap e133: 8 total, 7 up, 8 in 2024-09-21T12:35:43.830 INFO:teuthology.orchestra.run.smithi078.stdout: 5 234977 9.45 MB/sec execute 501 sec latency 47.444 ms 2024-09-21T12:35:44.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:43 smithi148 ceph-mon[49387]: osdmap e133: 8 total, 7 up, 8 in 2024-09-21T12:35:44.051 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:43 smithi148 podman[54991]: 2024-09-21 12:35:43.563685898 +0000 UTC m=+0.426712395 container init 6d7810eb0e7911c7df49bf806eae1780904a5ca29b50b9043aefb3d80db4a7da (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0842-4417849, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-21T12:35:44.051 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:43 smithi148 podman[54991]: 2024-09-21 12:35:43.567284254 +0000 UTC m=+0.430310751 container start 6d7810eb0e7911c7df49bf806eae1780904a5ca29b50b9043aefb3d80db4a7da (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5-deactivate, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T12:35:44.051 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:43 smithi148 podman[54991]: 2024-09-21 12:35:43.671918895 +0000 UTC m=+0.534945406 container attach 6d7810eb0e7911c7df49bf806eae1780904a5ca29b50b9043aefb3d80db4a7da (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True) 2024-09-21T12:35:44.051 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:43 smithi148 podman[54991]: 2024-09-21 12:35:43.763358755 +0000 UTC m=+0.626385252 container died 6d7810eb0e7911c7df49bf806eae1780904a5ca29b50b9043aefb3d80db4a7da (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0842-4417849, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-21T12:35:44.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:44 smithi078 ceph-mon[63866]: pgmap v236: 97 pgs: 10 peering, 5 stale+active+clean, 2 active+undersized+degraded, 80 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.9 MiB/s wr, 379 op/s; 5/1650 objects degraded (0.303%) 2024-09-21T12:35:44.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:44 smithi078 ceph-mon[63866]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T12:35:44.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:44 smithi078 ceph-mon[63866]: Health check failed: Degraded data redundancy: 5/1650 objects degraded (0.303%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T12:35:44.709 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:44 smithi148 ceph-mon[49387]: pgmap v236: 97 pgs: 10 peering, 5 stale+active+clean, 2 active+undersized+degraded, 80 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.9 MiB/s wr, 379 op/s; 5/1650 objects degraded (0.303%) 2024-09-21T12:35:44.710 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:44 smithi148 ceph-mon[49387]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T12:35:44.710 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:44 smithi148 ceph-mon[49387]: Health check failed: Degraded data redundancy: 5/1650 objects degraded (0.303%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T12:35:44.710 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:44 smithi148 podman[55014]: 2024-09-21 12:35:44.451777004 +0000 UTC m=+0.681310928 container remove 6d7810eb0e7911c7df49bf806eae1780904a5ca29b50b9043aefb3d80db4a7da (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5-deactivate, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-21T12:35:44.710 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:44 smithi148 systemd[1]: ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.5.service: Deactivated successfully. 2024-09-21T12:35:44.710 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:44 smithi148 systemd[1]: Stopped Ceph osd.5 for 6263deca-7813-11ef-baf6-efdc52797490. 2024-09-21T12:35:44.710 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:44 smithi148 systemd[1]: ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.5.service: Consumed 4min 34.380s CPU time. 2024-09-21T12:35:44.830 INFO:teuthology.orchestra.run.smithi078.stdout: 5 235267 9.44 MB/sec execute 502 sec latency 56.262 ms 2024-09-21T12:35:45.050 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:44 smithi148 systemd[1]: Starting Ceph osd.5 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:35:45.050 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:44 smithi148 podman[55110]: 2024-09-21 12:35:44.837856866 +0000 UTC m=+0.017928469 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:35:45.050 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:44 smithi148 podman[55110]: 2024-09-21 12:35:44.952972798 +0000 UTC m=+0.133044393 container create 2d52034336bf4e90eacb33f7a1bad3ffef11f5b80591717c7eb47a716251427f (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5-activate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-21T12:35:45.519 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:45 smithi148 podman[55110]: 2024-09-21 12:35:45.372515043 +0000 UTC m=+0.552586646 container init 2d52034336bf4e90eacb33f7a1bad3ffef11f5b80591717c7eb47a716251427f (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5-activate, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-21T12:35:45.519 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:45 smithi148 podman[55110]: 2024-09-21 12:35:45.376961717 +0000 UTC m=+0.557033316 container start 2d52034336bf4e90eacb33f7a1bad3ffef11f5b80591717c7eb47a716251427f (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5-activate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True) 2024-09-21T12:35:45.519 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:45 smithi148 podman[55110]: 2024-09-21 12:35:45.480879219 +0000 UTC m=+0.660950830 container attach 2d52034336bf4e90eacb33f7a1bad3ffef11f5b80591717c7eb47a716251427f (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-21T12:35:45.800 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:45 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5-activate[55124]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:35:45.800 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:45 smithi148 bash[55110]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:35:45.800 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:45 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5-activate[55124]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:35:45.800 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:45 smithi148 bash[55110]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:35:45.830 INFO:teuthology.orchestra.run.smithi078.stdout: 5 235489 9.44 MB/sec execute 503 sec latency 50.737 ms 2024-09-21T12:35:46.006 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:46.007 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:35:46.007 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:46.007 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:45 smithi078 ceph-mon[63866]: pgmap v237: 97 pgs: 2 active+undersized, 10 peering, 2 stale+active+clean, 6 active+undersized+degraded, 77 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 3.6 MiB/s wr, 347 op/s; 43/1647 objects degraded (2.611%) 2024-09-21T12:35:46.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:46.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:35:46.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:46.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:45 smithi148 ceph-mon[49387]: pgmap v237: 97 pgs: 2 active+undersized, 10 peering, 2 stale+active+clean, 6 active+undersized+degraded, 77 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 3.6 MiB/s wr, 347 op/s; 43/1647 objects degraded (2.611%) 2024-09-21T12:35:46.830 INFO:teuthology.orchestra.run.smithi078.stdout: 5 235661 9.42 MB/sec execute 504 sec latency 68.857 ms 2024-09-21T12:35:46.964 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-21T12:35:47.796 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:47 smithi078 ceph-mon[63866]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T12:35:47.830 INFO:teuthology.orchestra.run.smithi078.stdout: 5 235895 9.41 MB/sec execute 505 sec latency 59.173 ms 2024-09-21T12:35:47.965 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T12:35:47.965 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (8m) 73s ago 16m 20.0M - 0.27.0 11f11916f8cd e6e67ab07dec 2024-09-21T12:35:47.965 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (4m) 73s ago 16m 6895k - 19.3.0-5104-g44178494 b415816db8b4 46a10ea0b111 2024-09-21T12:35:47.965 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi148 smithi148 running (4m) 32s ago 16m 6895k - 19.3.0-5104-g44178494 b415816db8b4 23deb0615621 2024-09-21T12:35:47.965 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (7m) 73s ago 16m 53.3M - dad864ee21e9 2a140c36091b 2024-09-21T12:35:47.965 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.ytpzwh smithi078 running (10m) 73s ago 10m 38.4M - 17.2.0 e1d6a67b021e 5e9b109502a2 2024-09-21T12:35:47.965 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi148.uukezr smithi148 running (10m) 32s ago 10m 14.5M - 17.2.0 e1d6a67b021e b7e8aa0f3a6c 2024-09-21T12:35:47.965 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.ntwyan smithi078 *:8443,9283,8765 running (7m) 73s ago 18m 519M - 19.3.0-5104-g44178494 b415816db8b4 a61baedebf43 2024-09-21T12:35:47.965 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi148.oyiqmx smithi148 *:8443,9283,8765 running (6m) 32s ago 16m 457M - 19.3.0-5104-g44178494 b415816db8b4 0255535c6fdb 2024-09-21T12:35:47.965 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (5m) 73s ago 18m 55.9M 2048M 19.3.0-5104-g44178494 b415816db8b4 22b3097efbd8 2024-09-21T12:35:47.966 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi148 smithi148 running (5m) 32s ago 16m 42.3M 2048M 19.3.0-5104-g44178494 b415816db8b4 df8b08509136 2024-09-21T12:35:47.966 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.ydbhru smithi078 *:2049 running (10m) 73s ago 10m 279M - 4.0 e1d6a67b021e 7e5b0274ac38 2024-09-21T12:35:47.966 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi148.einuzt smithi148 *:2049 running (10m) 32s ago 10m 87.3M - 4.0 e1d6a67b021e 76a8b0eabede 2024-09-21T12:35:47.966 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (7m) 73s ago 16m 11.0M - 1.7.0 72c9c2088986 301adb4e9e22 2024-09-21T12:35:47.966 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi148 smithi148 *:9100 running (7m) 32s ago 16m 11.2M - 1.7.0 72c9c2088986 0cceff4bf342 2024-09-21T12:35:47.966 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (3m) 73s ago 15m 345M 1936M 19.3.0-5104-g44178494 b415816db8b4 7374add97534 2024-09-21T12:35:47.966 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (3m) 73s ago 14m 276M 1936M 19.3.0-5104-g44178494 b415816db8b4 14655a8da4e5 2024-09-21T12:35:47.966 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (2m) 73s ago 14m 219M 1936M 19.3.0-5104-g44178494 b415816db8b4 07ceda32f383 2024-09-21T12:35:47.966 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (84s) 73s ago 14m 11.4M 1936M 19.3.0-5104-g44178494 b415816db8b4 ead87e4306ff 2024-09-21T12:35:47.966 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi148 running (36s) 32s ago 13m 14.2M 2704M 19.3.0-5104-g44178494 b415816db8b4 10ec7d6b476e 2024-09-21T12:35:47.966 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi148 running (13m) 32s ago 13m 532M 2704M 17.2.0 e1d6a67b021e 7b44be78baca 2024-09-21T12:35:47.966 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi148 running (12m) 32s ago 12m 470M 2704M 17.2.0 e1d6a67b021e 3d23057910ea 2024-09-21T12:35:47.966 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi148 running (12m) 32s ago 12m 664M 2704M 17.2.0 e1d6a67b021e f9e4db9d35fa 2024-09-21T12:35:47.966 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (6m) 73s ago 15m 55.8M - 2.51.0 1d3b7f56885b face2d4cdd1b 2024-09-21T12:35:48.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:47 smithi148 ceph-mon[49387]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T12:35:48.564 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:35:48.564 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-21T12:35:48.564 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:35:48.564 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:35:48.564 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-21T12:35:48.564 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:35:48.564 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:35:48.564 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-21T12:35:48.564 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-21T12:35:48.564 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 5 2024-09-21T12:35:48.564 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:35:48.564 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-21T12:35:48.565 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:35:48.565 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:35:48.565 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-21T12:35:48.565 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-21T12:35:48.565 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 9 2024-09-21T12:35:48.565 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T12:35:48.565 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:35:48.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:48 smithi148 ceph-mon[49387]: from='client.44279 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:35:48.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:48 smithi148 ceph-mon[49387]: from='client.34380 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:35:48.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:48 smithi148 ceph-mon[49387]: pgmap v238: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 5.7 MiB/s wr, 550 op/s; 181/1653 objects degraded (10.950%) 2024-09-21T12:35:48.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:48 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/1875557294' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:35:48.801 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:48 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5-activate[55124]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T12:35:48.801 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:48 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5-activate[55124]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:35:48.801 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:48 smithi148 bash[55110]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T12:35:48.801 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:48 smithi148 bash[55110]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:35:48.801 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:48 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5-activate[55124]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:35:48.801 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:48 smithi148 bash[55110]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:35:48.801 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:48 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5-activate[55124]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T12:35:48.801 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:48 smithi148 bash[55110]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T12:35:48.801 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:48 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5-activate[55124]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-5 --no-mon-config 2024-09-21T12:35:48.802 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:48 smithi148 bash[55110]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-5 --no-mon-config 2024-09-21T12:35:48.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:48 smithi078 ceph-mon[63866]: from='client.44279 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:35:48.811 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:48 smithi078 ceph-mon[63866]: from='client.34380 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:35:48.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:48 smithi078 ceph-mon[63866]: pgmap v238: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 5.7 MiB/s wr, 550 op/s; 181/1653 objects degraded (10.950%) 2024-09-21T12:35:48.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:48 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/1875557294' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:35:48.830 INFO:teuthology.orchestra.run.smithi078.stdout: 5 236183 9.41 MB/sec execute 506 sec latency 64.066 ms 2024-09-21T12:35:49.202 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:35:49.202 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5", 2024-09-21T12:35:49.202 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-21T12:35:49.202 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T12:35:49.202 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-21T12:35:49.202 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-21T12:35:49.203 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-09-21T12:35:49.203 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr" 2024-09-21T12:35:49.203 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-21T12:35:49.203 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "11/23 daemons upgraded", 2024-09-21T12:35:49.203 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T12:35:49.203 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-21T12:35:49.203 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:35:49.269 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:48 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5-activate[55124]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-21T12:35:49.269 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:48 smithi148 bash[55110]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-21T12:35:49.269 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:48 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5-activate[55124]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-5/block 2024-09-21T12:35:49.269 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:48 smithi148 bash[55110]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-5/block 2024-09-21T12:35:49.269 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:48 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5-activate[55124]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T12:35:49.269 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:48 smithi148 bash[55110]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T12:35:49.269 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:48 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5-activate[55124]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T12:35:49.269 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:48 smithi148 bash[55110]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T12:35:49.269 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:48 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5-activate[55124]: --> ceph-volume lvm activate successful for osd ID: 5 2024-09-21T12:35:49.269 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:48 smithi148 bash[55110]: --> ceph-volume lvm activate successful for osd ID: 5 2024-09-21T12:35:49.270 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:48 smithi148 podman[55110]: 2024-09-21 12:35:48.898051733 +0000 UTC m=+4.078123332 container died 2d52034336bf4e90eacb33f7a1bad3ffef11f5b80591717c7eb47a716251427f (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5-activate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-21T12:35:49.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:49 smithi148 ceph-mon[49387]: from='client.34384 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:35:49.800 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:49 smithi148 podman[55328]: 2024-09-21 12:35:49.694830501 +0000 UTC m=+0.788852486 container remove 2d52034336bf4e90eacb33f7a1bad3ffef11f5b80591717c7eb47a716251427f (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5-activate, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T12:35:49.819 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:49 smithi078 ceph-mon[63866]: from='client.34384 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:35:49.830 INFO:teuthology.orchestra.run.smithi078.stdout: 5 236519 9.40 MB/sec execute 507 sec latency 50.156 ms 2024-09-21T12:35:49.960 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 180/1650 objects degraded (10.909%), 23 pgs degraded 2024-09-21T12:35:49.960 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-21T12:35:49.960 INFO:teuthology.orchestra.run.smithi078.stdout: osd.5 (root=default,host=smithi148) is down 2024-09-21T12:35:49.960 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 180/1650 objects degraded (10.909%), 23 pgs degraded 2024-09-21T12:35:49.960 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.2 is active+undersized+degraded, acting [1,6] 2024-09-21T12:35:49.961 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.3 is active+undersized+degraded, acting [2,7] 2024-09-21T12:35:49.961 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.b is active+undersized+degraded, acting [7,4] 2024-09-21T12:35:49.961 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.c is active+undersized+degraded, acting [2,0] 2024-09-21T12:35:49.961 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.12 is active+undersized+degraded, acting [3,7] 2024-09-21T12:35:49.961 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.14 is active+undersized+degraded, acting [6,3] 2024-09-21T12:35:49.961 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.15 is active+undersized+degraded, acting [1,0] 2024-09-21T12:35:49.961 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.16 is active+undersized+degraded, acting [6,2] 2024-09-21T12:35:49.961 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.17 is active+undersized+degraded, acting [6,2] 2024-09-21T12:35:49.961 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.18 is active+undersized+degraded, acting [4,2] 2024-09-21T12:35:49.961 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1c is active+undersized+degraded, acting [7,2] 2024-09-21T12:35:49.961 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.1e is active+undersized+degraded, acting [3,0] 2024-09-21T12:35:49.961 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.2 is active+undersized+degraded, acting [3,6] 2024-09-21T12:35:49.961 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.4 is active+undersized+degraded, acting [1,2] 2024-09-21T12:35:49.962 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.5 is active+undersized+degraded, acting [3,2] 2024-09-21T12:35:49.962 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.c is active+undersized+degraded, acting [3,6] 2024-09-21T12:35:49.962 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.d is active+undersized+degraded, acting [7,6] 2024-09-21T12:35:49.962 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.10 is active+undersized+degraded, acting [6,0] 2024-09-21T12:35:49.962 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.16 is active+undersized+degraded, acting [7,1] 2024-09-21T12:35:49.962 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.17 is active+undersized+degraded, acting [0,3] 2024-09-21T12:35:49.962 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1c is active+undersized+degraded, acting [4,1] 2024-09-21T12:35:49.962 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1d is active+undersized+degraded, acting [4,6] 2024-09-21T12:35:49.962 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.1f is active+undersized+degraded, acting [0,2] 2024-09-21T12:35:50.300 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:49 smithi148 podman[55387]: 2024-09-21 12:35:49.814470614 +0000 UTC m=+0.017993118 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:35:50.300 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:49 smithi148 podman[55387]: 2024-09-21 12:35:49.937583696 +0000 UTC m=+0.141106184 container create 06ac02700ea142595989cba71ee709466fa5661cf7cdc5175bd2baba234062a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-21T12:35:50.663 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:50 smithi148 podman[55387]: 2024-09-21 12:35:50.365812544 +0000 UTC m=+0.569335038 container init 06ac02700ea142595989cba71ee709466fa5661cf7cdc5175bd2baba234062a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T12:35:50.663 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:50 smithi148 podman[55387]: 2024-09-21 12:35:50.369272 +0000 UTC m=+0.572794489 container start 06ac02700ea142595989cba71ee709466fa5661cf7cdc5175bd2baba234062a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-21T12:35:50.663 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:50 smithi148 ceph-osd[55405]: -- 172.21.15.148:0/1133048016 <== mon.1 v2:172.21.15.148:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d3577ee1e0 con 0x55d35777a000 2024-09-21T12:35:50.663 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:50 smithi148 bash[55387]: 06ac02700ea142595989cba71ee709466fa5661cf7cdc5175bd2baba234062a5 2024-09-21T12:35:50.663 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:50 smithi148 systemd[1]: Started Ceph osd.5 for 6263deca-7813-11ef-baf6-efdc52797490. 2024-09-21T12:35:50.830 INFO:teuthology.orchestra.run.smithi078.stdout: 5 236777 9.39 MB/sec execute 508 sec latency 105.960 ms 2024-09-21T12:35:50.859 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:50 smithi078 ceph-mon[63866]: from='client.44291 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:35:50.859 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:50 smithi078 ceph-mon[63866]: pgmap v239: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.4 MiB/s wr, 431 op/s; 180/1650 objects degraded (10.909%) 2024-09-21T12:35:50.859 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:50 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 180/1650 objects degraded (10.909%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T12:35:50.859 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:50 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/1880328841' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:35:50.859 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:50 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:50.859 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:50 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:50.859 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:50 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:35:51.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:50 smithi148 ceph-mon[49387]: from='client.44291 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:35:51.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:50 smithi148 ceph-mon[49387]: pgmap v239: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.4 MiB/s wr, 431 op/s; 180/1650 objects degraded (10.909%) 2024-09-21T12:35:51.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:50 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 180/1650 objects degraded (10.909%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T12:35:51.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:50 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/1880328841' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:35:51.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:50 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:51.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:50 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:51.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:50 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:35:51.830 INFO:teuthology.orchestra.run.smithi078.stdout: 5 237093 9.39 MB/sec execute 509 sec latency 63.857 ms 2024-09-21T12:35:52.300 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:35:52 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:35:52.265+0000 7f841dd13640 -1 Falling back to public interface 2024-09-21T12:35:52.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:52 smithi148 ceph-mon[49387]: pgmap v240: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.4 MiB/s wr, 463 op/s; 178/1590 objects degraded (11.195%) 2024-09-21T12:35:52.830 INFO:teuthology.orchestra.run.smithi078.stdout: 5 237414 9.39 MB/sec execute 510 sec latency 79.505 ms 2024-09-21T12:35:52.913 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:52 smithi078 ceph-mon[63866]: pgmap v240: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.4 MiB/s wr, 463 op/s; 178/1590 objects degraded (11.195%) 2024-09-21T12:35:53.830 INFO:teuthology.orchestra.run.smithi078.stdout: 5 237720 9.39 MB/sec execute 511 sec latency 69.615 ms 2024-09-21T12:35:54.698 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:54 smithi078 ceph-mon[63866]: pgmap v241: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.8 MiB/s wr, 453 op/s; 165/1542 objects degraded (10.700%) 2024-09-21T12:35:54.699 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:54 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:54.699 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:54 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:54.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 237959 9.39 MB/sec execute 512 sec latency 63.675 ms 2024-09-21T12:35:55.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:54 smithi148 ceph-mon[49387]: pgmap v241: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.8 MiB/s wr, 453 op/s; 165/1542 objects degraded (10.700%) 2024-09-21T12:35:55.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:54 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:55.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:54 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:55.830 INFO:teuthology.orchestra.run.smithi078.stdout: 5 238261 9.40 MB/sec execute 513 sec latency 124.538 ms 2024-09-21T12:35:56.223 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:56 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:56.223 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:56 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:56.223 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:56 smithi078 ceph-mon[63866]: pgmap v242: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.6 MiB/s wr, 353 op/s; 167/1563 objects degraded (10.685%) 2024-09-21T12:35:56.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:56 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:56.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:56 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:35:56.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:56 smithi148 ceph-mon[49387]: pgmap v242: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.6 MiB/s wr, 353 op/s; 167/1563 objects degraded (10.685%) 2024-09-21T12:35:56.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 238564 9.40 MB/sec execute 514 sec latency 75.243 ms 2024-09-21T12:35:57.249 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:57 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 167/1563 objects degraded (10.685%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T12:35:57.264 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:57 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 167/1563 objects degraded (10.685%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T12:35:57.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 238822 9.39 MB/sec execute 515 sec latency 72.563 ms 2024-09-21T12:35:58.274 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:35:58 smithi078 ceph-mon[63866]: pgmap v243: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.1 MiB/s wr, 513 op/s; 177/1665 objects degraded (10.631%) 2024-09-21T12:35:58.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:35:58 smithi148 ceph-mon[49387]: pgmap v243: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.1 MiB/s wr, 513 op/s; 177/1665 objects degraded (10.631%) 2024-09-21T12:35:58.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 239069 9.38 MB/sec execute 516 sec latency 40.913 ms 2024-09-21T12:35:59.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 239286 9.38 MB/sec execute 517 sec latency 85.096 ms 2024-09-21T12:36:00.334 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:00.334 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:00.334 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:36:00.334 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:36:00.334 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:00.334 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:00.334 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:00 smithi078 ceph-mon[63866]: pgmap v244: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 3.5 MiB/s wr, 346 op/s; 177/1665 objects degraded (10.631%) 2024-09-21T12:36:00.334 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:36:00.334 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:36:00.334 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:36:00.334 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:36:00.335 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-21T12:36:00.335 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:00 smithi078 ceph-mon[63866]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T12:36:00.335 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:00 smithi078 ceph-mon[63866]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-21T12:36:00.335 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:00.335 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:36:00.533 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:00.533 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:00.533 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:36:00.533 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:36:00.533 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:00.533 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:00.533 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:00 smithi148 ceph-mon[49387]: pgmap v244: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 3.5 MiB/s wr, 346 op/s; 177/1665 objects degraded (10.631%) 2024-09-21T12:36:00.533 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:36:00.533 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:36:00.533 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:36:00.533 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:36:00.533 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-21T12:36:00.534 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:00 smithi148 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T12:36:00.534 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:00 smithi148 ceph-mon[49387]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-21T12:36:00.534 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:00.534 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:36:00.800 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:36:00 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:36:00.530+0000 7f841dd13640 -1 osd.5 0 read_superblock omap replica is missing. 2024-09-21T12:36:00.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 239541 9.36 MB/sec execute 518 sec latency 92.687 ms 2024-09-21T12:36:01.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 239853 9.36 MB/sec execute 519 sec latency 74.883 ms 2024-09-21T12:36:01.870 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:01 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 172/1605 objects degraded (10.717%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T12:36:02.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:01 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 172/1605 objects degraded (10.717%), 23 pgs degraded (PG_DEGRADED) 2024-09-21T12:36:02.050 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:36:01 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:36:01.785+0000 7f841dd13640 -1 osd.5 131 log_to_monitors true 2024-09-21T12:36:02.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 240067 9.35 MB/sec execute 520 sec latency 105.884 ms 2024-09-21T12:36:02.907 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:02 smithi078 ceph-mon[63866]: pgmap v245: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.5 MiB/s wr, 459 op/s; 172/1605 objects degraded (10.717%) 2024-09-21T12:36:02.908 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:02 smithi078 ceph-mon[63866]: from='osd.5 [v2:172.21.15.148:6808/2173225402,v1:172.21.15.148:6809/2173225402]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T12:36:02.908 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:02 smithi078 ceph-mon[63866]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T12:36:03.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:02 smithi148 ceph-mon[49387]: pgmap v245: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.5 MiB/s wr, 459 op/s; 172/1605 objects degraded (10.717%) 2024-09-21T12:36:03.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:02 smithi148 ceph-mon[49387]: from='osd.5 [v2:172.21.15.148:6808/2173225402,v1:172.21.15.148:6809/2173225402]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T12:36:03.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:02 smithi148 ceph-mon[49387]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T12:36:03.550 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:36:03 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:36:03.188+0000 7f8411ab5640 -1 osd.5 131 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T12:36:03.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 240299 9.34 MB/sec execute 521 sec latency 153.701 ms 2024-09-21T12:36:03.918 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:03 smithi078 ceph-mon[63866]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T12:36:03.918 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:03 smithi078 ceph-mon[63866]: osdmap e134: 8 total, 7 up, 8 in 2024-09-21T12:36:03.918 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:03 smithi078 ceph-mon[63866]: from='osd.5 [v2:172.21.15.148:6808/2173225402,v1:172.21.15.148:6809/2173225402]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi148", "root=default"]} : dispatch 2024-09-21T12:36:03.918 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:03 smithi078 ceph-mon[63866]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi148", "root=default"]} : dispatch 2024-09-21T12:36:03.918 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:03 smithi078 ceph-mon[63866]: from='osd.5 ' entity='osd.5' 2024-09-21T12:36:04.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:03 smithi148 ceph-mon[49387]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T12:36:04.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:03 smithi148 ceph-mon[49387]: osdmap e134: 8 total, 7 up, 8 in 2024-09-21T12:36:04.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:03 smithi148 ceph-mon[49387]: from='osd.5 [v2:172.21.15.148:6808/2173225402,v1:172.21.15.148:6809/2173225402]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi148", "root=default"]} : dispatch 2024-09-21T12:36:04.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:03 smithi148 ceph-mon[49387]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi148", "root=default"]} : dispatch 2024-09-21T12:36:04.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:03 smithi148 ceph-mon[49387]: from='osd.5 ' entity='osd.5' 2024-09-21T12:36:04.711 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:04 smithi078 ceph-mon[63866]: pgmap v247: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 327 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.3 MiB/s wr, 395 op/s; 170/1575 objects degraded (10.794%) 2024-09-21T12:36:04.712 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:04 smithi078 ceph-mon[63866]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T12:36:04.712 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:04 smithi078 ceph-mon[63866]: osd.5 [v2:172.21.15.148:6808/2173225402,v1:172.21.15.148:6809/2173225402] boot 2024-09-21T12:36:04.712 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:04 smithi078 ceph-mon[63866]: osdmap e135: 8 total, 8 up, 8 in 2024-09-21T12:36:04.712 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:04 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T12:36:04.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 240467 9.33 MB/sec execute 522 sec latency 511.006 ms 2024-09-21T12:36:05.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:04 smithi148 ceph-mon[49387]: pgmap v247: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 327 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.3 MiB/s wr, 395 op/s; 170/1575 objects degraded (10.794%) 2024-09-21T12:36:05.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:04 smithi148 ceph-mon[49387]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T12:36:05.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:04 smithi148 ceph-mon[49387]: osd.5 [v2:172.21.15.148:6808/2173225402,v1:172.21.15.148:6809/2173225402] boot 2024-09-21T12:36:05.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:04 smithi148 ceph-mon[49387]: osdmap e135: 8 total, 8 up, 8 in 2024-09-21T12:36:05.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:04 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T12:36:05.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 240630 9.32 MB/sec execute 523 sec latency 1511.033 ms 2024-09-21T12:36:06.449 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:06 smithi078 ceph-mon[63866]: osdmap e136: 8 total, 8 up, 8 in 2024-09-21T12:36:06.449 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:06 smithi078 ceph-mon[63866]: pgmap v250: 97 pgs: 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 7 active+undersized, 17 active+undersized+degraded, 71 active+clean; 325 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 3.7 MiB/s wr, 474 op/s; 152/1557 objects degraded (9.762%); 3.7 KiB/s, 1 objects/s recovering 2024-09-21T12:36:06.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:06 smithi148 ceph-mon[49387]: osdmap e136: 8 total, 8 up, 8 in 2024-09-21T12:36:06.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:06 smithi148 ceph-mon[49387]: pgmap v250: 97 pgs: 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 7 active+undersized, 17 active+undersized+degraded, 71 active+clean; 325 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 3.7 MiB/s wr, 474 op/s; 152/1557 objects degraded (9.762%); 3.7 KiB/s, 1 objects/s recovering 2024-09-21T12:36:06.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 240932 9.33 MB/sec execute 524 sec latency 1968.995 ms 2024-09-21T12:36:07.462 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:07 smithi078 ceph-mon[63866]: osdmap e137: 8 total, 8 up, 8 in 2024-09-21T12:36:07.462 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:07 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 152/1557 objects degraded (9.762%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T12:36:07.463 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:07 smithi078 ceph-mon[63866]: osdmap e138: 8 total, 8 up, 8 in 2024-09-21T12:36:07.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:07 smithi148 ceph-mon[49387]: osdmap e137: 8 total, 8 up, 8 in 2024-09-21T12:36:07.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:07 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 152/1557 objects degraded (9.762%), 19 pgs degraded (PG_DEGRADED) 2024-09-21T12:36:07.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:07 smithi148 ceph-mon[49387]: osdmap e138: 8 total, 8 up, 8 in 2024-09-21T12:36:07.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 241302 9.33 MB/sec execute 525 sec latency 409.335 ms 2024-09-21T12:36:08.505 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:08 smithi078 ceph-mon[63866]: pgmap v253: 97 pgs: 1 active+undersized+remapped, 2 remapped+peering, 1 active+recovering+degraded, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 2 active+undersized, 10 active+undersized+degraded, 78 active+clean; 324 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.2 MiB/s wr, 384 op/s; 99/1515 objects degraded (6.535%); 27 KiB/s, 16 objects/s recovering 2024-09-21T12:36:08.506 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:08 smithi078 ceph-mon[63866]: osdmap e139: 8 total, 8 up, 8 in 2024-09-21T12:36:08.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:08 smithi148 ceph-mon[49387]: pgmap v253: 97 pgs: 1 active+undersized+remapped, 2 remapped+peering, 1 active+recovering+degraded, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 2 active+undersized, 10 active+undersized+degraded, 78 active+clean; 324 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.2 MiB/s wr, 384 op/s; 99/1515 objects degraded (6.535%); 27 KiB/s, 16 objects/s recovering 2024-09-21T12:36:08.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:08 smithi148 ceph-mon[49387]: osdmap e139: 8 total, 8 up, 8 in 2024-09-21T12:36:08.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 241553 9.33 MB/sec execute 526 sec latency 592.657 ms 2024-09-21T12:36:09.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 241748 9.32 MB/sec execute 527 sec latency 847.427 ms 2024-09-21T12:36:10.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:10 smithi078 ceph-mon[63866]: osdmap e140: 8 total, 8 up, 8 in 2024-09-21T12:36:10.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:10 smithi078 ceph-mon[63866]: pgmap v256: 97 pgs: 3 peering, 2 remapped+peering, 1 active+recovering+degraded, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 88 active+clean; 327 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 2.6 MiB/s wr, 265 op/s; 33/1533 objects degraded (2.153%); 76 KiB/s, 21 objects/s recovering 2024-09-21T12:36:10.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:10 smithi148 ceph-mon[49387]: osdmap e140: 8 total, 8 up, 8 in 2024-09-21T12:36:10.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:10 smithi148 ceph-mon[49387]: pgmap v256: 97 pgs: 3 peering, 2 remapped+peering, 1 active+recovering+degraded, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 88 active+clean; 327 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 2.6 MiB/s wr, 265 op/s; 33/1533 objects degraded (2.153%); 76 KiB/s, 21 objects/s recovering 2024-09-21T12:36:10.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 241946 9.32 MB/sec execute 528 sec latency 1338.742 ms 2024-09-21T12:36:11.088 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:11 smithi078 ceph-mon[63866]: osdmap e141: 8 total, 8 up, 8 in 2024-09-21T12:36:11.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:11 smithi148 ceph-mon[49387]: osdmap e141: 8 total, 8 up, 8 in 2024-09-21T12:36:11.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 242216 9.32 MB/sec execute 529 sec latency 48.718 ms 2024-09-21T12:36:12.132 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:12 smithi078 ceph-mon[63866]: pgmap v258: 97 pgs: 3 peering, 1 active+recovering+undersized+degraded+remapped, 93 active+clean; 332 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.0 MiB/s wr, 248 op/s; 8/1614 objects degraded (0.496%); 864 KiB/s, 22 objects/s recovering 2024-09-21T12:36:12.132 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:12 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 33/1533 objects degraded (2.153%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T12:36:12.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:12 smithi148 ceph-mon[49387]: pgmap v258: 97 pgs: 3 peering, 1 active+recovering+undersized+degraded+remapped, 93 active+clean; 332 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.0 MiB/s wr, 248 op/s; 8/1614 objects degraded (0.496%); 864 KiB/s, 22 objects/s recovering 2024-09-21T12:36:12.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:12 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 33/1533 objects degraded (2.153%), 4 pgs degraded (PG_DEGRADED) 2024-09-21T12:36:12.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 242506 9.31 MB/sec execute 530 sec latency 61.674 ms 2024-09-21T12:36:13.665 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:13 smithi078 ceph-mon[63866]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8/1614 objects degraded (0.496%), 1 pg degraded) 2024-09-21T12:36:13.666 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:13 smithi078 ceph-mon[63866]: Cluster is now healthy 2024-09-21T12:36:13.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 242794 9.31 MB/sec execute 531 sec latency 75.404 ms 2024-09-21T12:36:14.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:13 smithi148 ceph-mon[49387]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8/1614 objects degraded (0.496%), 1 pg degraded) 2024-09-21T12:36:14.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:13 smithi148 ceph-mon[49387]: Cluster is now healthy 2024-09-21T12:36:14.718 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:14 smithi078 ceph-mon[63866]: pgmap v259: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.8 KiB/s rd, 4.7 MiB/s wr, 450 op/s; 1.5 MiB/s, 27 objects/s recovering 2024-09-21T12:36:14.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 243020 9.29 MB/sec execute 532 sec latency 113.724 ms 2024-09-21T12:36:15.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:14 smithi148 ceph-mon[49387]: pgmap v259: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.8 KiB/s rd, 4.7 MiB/s wr, 450 op/s; 1.5 MiB/s, 27 objects/s recovering 2024-09-21T12:36:15.753 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:15 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-21T12:36:15.753 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:15 smithi078 ceph-mon[63866]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T12:36:15.753 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:15 smithi078 ceph-mon[63866]: Upgrade: osd.6 is safe to restart 2024-09-21T12:36:15.753 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:15 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:15.753 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:15 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:36:15.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:15 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-21T12:36:15.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:15 smithi148 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T12:36:15.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:15 smithi148 ceph-mon[49387]: Upgrade: osd.6 is safe to restart 2024-09-21T12:36:15.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:15 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:15.801 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:15 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:36:15.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 243184 9.28 MB/sec execute 533 sec latency 64.333 ms 2024-09-21T12:36:16.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 243410 9.28 MB/sec execute 534 sec latency 70.899 ms 2024-09-21T12:36:17.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:16 smithi078 ceph-mon[63866]: pgmap v260: 97 pgs: 97 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.2 KiB/s rd, 4.0 MiB/s wr, 362 op/s; 1.1 MiB/s, 18 objects/s recovering 2024-09-21T12:36:17.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:16 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:17.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:16 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T12:36:17.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:16 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:36:17.186 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:16 smithi148 ceph-mon[49387]: pgmap v260: 97 pgs: 97 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.2 KiB/s rd, 4.0 MiB/s wr, 362 op/s; 1.1 MiB/s, 18 objects/s recovering 2024-09-21T12:36:17.186 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:16 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:17.186 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:16 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T12:36:17.186 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:16 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:36:17.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 243710 9.28 MB/sec execute 535 sec latency 80.809 ms 2024-09-21T12:36:17.991 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:17 smithi148 ceph-mon[49387]: Upgrade: Updating osd.6 2024-09-21T12:36:17.991 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:17 smithi148 ceph-mon[49387]: Deploying daemon osd.6 on smithi148 2024-09-21T12:36:18.084 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:17 smithi078 ceph-mon[63866]: Upgrade: Updating osd.6 2024-09-21T12:36:18.084 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:17 smithi078 ceph-mon[63866]: Deploying daemon osd.6 on smithi148 2024-09-21T12:36:18.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 244037 9.27 MB/sec execute 536 sec latency 65.542 ms 2024-09-21T12:36:19.114 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:18 smithi078 ceph-mon[63866]: pgmap v261: 97 pgs: 97 active+clean; 328 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.3 MiB/s wr, 346 op/s; 1.0 MiB/s, 15 objects/s recovering 2024-09-21T12:36:19.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:18 smithi148 ceph-mon[49387]: pgmap v261: 97 pgs: 97 active+clean; 328 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.3 MiB/s wr, 346 op/s; 1.0 MiB/s, 15 objects/s recovering 2024-09-21T12:36:19.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 244351 9.27 MB/sec execute 537 sec latency 45.985 ms 2024-09-21T12:36:20.385 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-21T12:36:20.785 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:20 smithi148 systemd[1]: Stopping Ceph osd.6 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:36:20.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 244573 9.27 MB/sec execute 538 sec latency 67.861 ms 2024-09-21T12:36:20.968 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:20 smithi078 ceph-mon[63866]: pgmap v262: 97 pgs: 97 active+clean; 328 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 329 op/s; 884 KiB/s, 13 objects/s recovering 2024-09-21T12:36:20.968 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:20 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:20.968 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:20 smithi078 ceph-mon[63866]: osd.6 marked itself down and dead 2024-09-21T12:36:21.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:20 smithi148 ceph-mon[49387]: pgmap v262: 97 pgs: 97 active+clean; 328 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 329 op/s; 884 KiB/s, 13 objects/s recovering 2024-09-21T12:36:21.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:20 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:21.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:20 smithi148 ceph-mon[49387]: osd.6 marked itself down and dead 2024-09-21T12:36:21.051 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:20 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[35990]: 2024-09-21T12:36:20.782+0000 7f0f106cd700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T12:36:21.051 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:20 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[35990]: 2024-09-21T12:36:20.782+0000 7f0f106cd700 -1 osd.6 141 *** Got signal Terminated *** 2024-09-21T12:36:21.051 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:20 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[35990]: 2024-09-21T12:36:20.782+0000 7f0f106cd700 -1 osd.6 141 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T12:36:21.236 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T12:36:21.236 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (8m) 107s ago 17m 20.0M - 0.27.0 11f11916f8cd e6e67ab07dec 2024-09-21T12:36:21.236 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (5m) 107s ago 17m 6895k - 19.3.0-5104-g44178494 b415816db8b4 46a10ea0b111 2024-09-21T12:36:21.236 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi148 smithi148 running (5m) 26s ago 16m 6895k - 19.3.0-5104-g44178494 b415816db8b4 23deb0615621 2024-09-21T12:36:21.236 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (8m) 107s ago 17m 53.3M - dad864ee21e9 2a140c36091b 2024-09-21T12:36:21.236 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.ytpzwh smithi078 running (11m) 107s ago 11m 38.4M - 17.2.0 e1d6a67b021e 5e9b109502a2 2024-09-21T12:36:21.237 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi148.uukezr smithi148 running (11m) 26s ago 11m 14.6M - 17.2.0 e1d6a67b021e b7e8aa0f3a6c 2024-09-21T12:36:21.237 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.ntwyan smithi078 *:8443,9283,8765 running (7m) 107s ago 18m 519M - 19.3.0-5104-g44178494 b415816db8b4 a61baedebf43 2024-09-21T12:36:21.237 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi148.oyiqmx smithi148 *:8443,9283,8765 running (6m) 26s ago 16m 457M - 19.3.0-5104-g44178494 b415816db8b4 0255535c6fdb 2024-09-21T12:36:21.237 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (6m) 107s ago 18m 55.9M 2048M 19.3.0-5104-g44178494 b415816db8b4 22b3097efbd8 2024-09-21T12:36:21.237 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi148 smithi148 running (6m) 26s ago 16m 44.5M 2048M 19.3.0-5104-g44178494 b415816db8b4 df8b08509136 2024-09-21T12:36:21.237 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.ydbhru smithi078 *:2049 running (11m) 107s ago 11m 279M - 4.0 e1d6a67b021e 7e5b0274ac38 2024-09-21T12:36:21.237 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi148.einuzt smithi148 *:2049 running (11m) 26s ago 11m 87.3M - 4.0 e1d6a67b021e 76a8b0eabede 2024-09-21T12:36:21.237 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (8m) 107s ago 17m 11.0M - 1.7.0 72c9c2088986 301adb4e9e22 2024-09-21T12:36:21.237 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi148 smithi148 *:9100 running (7m) 26s ago 16m 11.3M - 1.7.0 72c9c2088986 0cceff4bf342 2024-09-21T12:36:21.237 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (4m) 107s ago 16m 345M 1936M 19.3.0-5104-g44178494 b415816db8b4 7374add97534 2024-09-21T12:36:21.237 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (3m) 107s ago 15m 276M 1936M 19.3.0-5104-g44178494 b415816db8b4 14655a8da4e5 2024-09-21T12:36:21.237 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (2m) 107s ago 15m 219M 1936M 19.3.0-5104-g44178494 b415816db8b4 07ceda32f383 2024-09-21T12:36:21.237 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (117s) 107s ago 14m 11.4M 1936M 19.3.0-5104-g44178494 b415816db8b4 ead87e4306ff 2024-09-21T12:36:21.237 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi148 running (70s) 26s ago 14m 257M 2704M 19.3.0-5104-g44178494 b415816db8b4 10ec7d6b476e 2024-09-21T12:36:21.238 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi148 running (31s) 26s ago 13m 12.1M 2704M 19.3.0-5104-g44178494 b415816db8b4 06ac02700ea1 2024-09-21T12:36:21.238 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi148 running (13m) 26s ago 13m 496M 2704M 17.2.0 e1d6a67b021e 3d23057910ea 2024-09-21T12:36:21.238 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi148 running (13m) 26s ago 13m 709M 2704M 17.2.0 e1d6a67b021e f9e4db9d35fa 2024-09-21T12:36:21.238 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (6m) 107s ago 16m 55.8M - 2.51.0 1d3b7f56885b face2d4cdd1b 2024-09-21T12:36:21.698 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:36:21.698 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-21T12:36:21.698 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:36:21.698 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:36:21.698 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-21T12:36:21.699 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:36:21.699 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:36:21.699 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-21T12:36:21.699 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-21T12:36:21.699 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 6 2024-09-21T12:36:21.699 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:36:21.700 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-21T12:36:21.700 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:36:21.700 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:36:21.700 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-21T12:36:21.700 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-21T12:36:21.700 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 10 2024-09-21T12:36:21.700 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T12:36:21.701 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:36:21.800 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:21 smithi148 podman[56985]: 2024-09-21 12:36:21.525969076 +0000 UTC m=+0.876805843 container died 3d23057910ea89195e7afbc468981fa69991c8123130b9738c0bbf1e6b279a4d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , RELEASE=HEAD, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, architecture=x86_64, release=754, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870) 2024-09-21T12:36:21.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 244752 9.26 MB/sec execute 539 sec latency 852.698 ms 2024-09-21T12:36:22.009 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:21 smithi078 ceph-mon[63866]: from='client.34402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:36:22.009 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:21 smithi078 ceph-mon[63866]: from='client.34406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:36:22.009 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:21 smithi078 ceph-mon[63866]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T12:36:22.009 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:21 smithi078 ceph-mon[63866]: osdmap e142: 8 total, 7 up, 8 in 2024-09-21T12:36:22.009 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:21 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/1793242286' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:36:22.139 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:36:22.139 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5", 2024-09-21T12:36:22.139 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-21T12:36:22.139 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T12:36:22.140 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-21T12:36:22.140 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-21T12:36:22.140 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-09-21T12:36:22.140 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr" 2024-09-21T12:36:22.140 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-21T12:36:22.140 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "12/23 daemons upgraded", 2024-09-21T12:36:22.140 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T12:36:22.140 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-21T12:36:22.140 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:36:22.277 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:21 smithi148 ceph-mon[49387]: from='client.34402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:36:22.278 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:21 smithi148 ceph-mon[49387]: from='client.34406 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:36:22.278 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:21 smithi148 ceph-mon[49387]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T12:36:22.278 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:21 smithi148 ceph-mon[49387]: osdmap e142: 8 total, 7 up, 8 in 2024-09-21T12:36:22.278 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:21 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/1793242286' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:36:22.278 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:22 smithi148 podman[56985]: 2024-09-21 12:36:22.041674198 +0000 UTC m=+1.392510952 container cleanup 3d23057910ea89195e7afbc468981fa69991c8123130b9738c0bbf1e6b279a4d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, io.openshift.tags=base centos centos-stream, vcs-type=git, com.redhat.component=centos-stream-container, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc.) 2024-09-21T12:36:22.278 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:22 smithi148 bash[56985]: ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6 2024-09-21T12:36:22.550 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:22 smithi148 podman[57003]: 2024-09-21 12:36:22.276929074 +0000 UTC m=+0.747344674 container remove 3d23057910ea89195e7afbc468981fa69991c8123130b9738c0bbf1e6b279a4d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, GIT_CLEAN=True, RELEASE=HEAD, ceph=True, distribution-scope=public, version=8, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=) 2024-09-21T12:36:22.644 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_WARN 1 osds down 2024-09-21T12:36:22.644 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-21T12:36:22.644 INFO:teuthology.orchestra.run.smithi078.stdout: osd.6 (root=default,host=smithi148) is down 2024-09-21T12:36:22.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 245059 9.26 MB/sec execute 540 sec latency 1024.051 ms 2024-09-21T12:36:23.019 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:22 smithi078 ceph-mon[63866]: from='client.34410 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:36:23.019 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:22 smithi078 ceph-mon[63866]: pgmap v264: 97 pgs: 1 peering, 9 stale+active+clean, 87 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.0 MiB/s wr, 447 op/s; 511 KiB/s, 6 objects/s recovering 2024-09-21T12:36:23.019 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:22 smithi078 ceph-mon[63866]: osdmap e143: 8 total, 7 up, 8 in 2024-09-21T12:36:23.019 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:22 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/3315592719' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:36:23.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:22 smithi148 ceph-mon[49387]: from='client.34410 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:36:23.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:22 smithi148 ceph-mon[49387]: pgmap v264: 97 pgs: 1 peering, 9 stale+active+clean, 87 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.0 MiB/s wr, 447 op/s; 511 KiB/s, 6 objects/s recovering 2024-09-21T12:36:23.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:22 smithi148 ceph-mon[49387]: osdmap e143: 8 total, 7 up, 8 in 2024-09-21T12:36:23.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:22 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/3315592719' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:36:23.051 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:22 smithi148 podman[57085]: 2024-09-21 12:36:22.695887065 +0000 UTC m=+0.020052240 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:36:23.051 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:22 smithi148 podman[57085]: 2024-09-21 12:36:22.819654087 +0000 UTC m=+0.143819234 container create ba3722bc16de6c0d33ecc8909f7b013355d420ae333ad2775297f371fad69926 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6-deactivate, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-21-0842-4417849, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-21T12:36:23.550 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:23 smithi148 podman[57085]: 2024-09-21 12:36:23.101587154 +0000 UTC m=+0.425752318 container init ba3722bc16de6c0d33ecc8909f7b013355d420ae333ad2775297f371fad69926 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T12:36:23.551 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:23 smithi148 podman[57085]: 2024-09-21 12:36:23.106222831 +0000 UTC m=+0.430387979 container start ba3722bc16de6c0d33ecc8909f7b013355d420ae333ad2775297f371fad69926 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6-deactivate, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-21-0842-4417849, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T12:36:23.551 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:23 smithi148 podman[57085]: 2024-09-21 12:36:23.201474119 +0000 UTC m=+0.525639277 container attach ba3722bc16de6c0d33ecc8909f7b013355d420ae333ad2775297f371fad69926 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-21-0842-4417849) 2024-09-21T12:36:23.551 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:23 smithi148 podman[57085]: 2024-09-21 12:36:23.34250687 +0000 UTC m=+0.666672016 container died ba3722bc16de6c0d33ecc8909f7b013355d420ae333ad2775297f371fad69926 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-21T12:36:23.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 245426 9.27 MB/sec execute 541 sec latency 47.001 ms 2024-09-21T12:36:24.042 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:23 smithi078 ceph-mon[63866]: from='client.34418 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:36:24.229 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:23 smithi148 ceph-mon[49387]: from='client.34418 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:36:24.230 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:23 smithi148 podman[57107]: 2024-09-21 12:36:23.983939794 +0000 UTC m=+0.633828197 container remove ba3722bc16de6c0d33ecc8909f7b013355d420ae333ad2775297f371fad69926 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6-deactivate, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T12:36:24.230 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:24 smithi148 systemd[1]: ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.6.service: Deactivated successfully. 2024-09-21T12:36:24.230 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:24 smithi148 systemd[1]: Stopped Ceph osd.6 for 6263deca-7813-11ef-baf6-efdc52797490. 2024-09-21T12:36:24.230 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:24 smithi148 systemd[1]: ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.6.service: Consumed 4min 25.390s CPU time. 2024-09-21T12:36:24.551 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:24 smithi148 systemd[1]: Starting Ceph osd.6 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:36:24.551 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:24 smithi148 podman[57207]: 2024-09-21 12:36:24.360134035 +0000 UTC m=+0.017944096 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:36:24.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 245819 9.28 MB/sec execute 542 sec latency 41.657 ms 2024-09-21T12:36:25.045 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:24 smithi148 ceph-mon[49387]: pgmap v266: 97 pgs: 2 active+undersized, 19 peering, 2 stale+active+clean, 5 active+undersized+degraded, 69 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.4 MiB/s wr, 388 op/s; 34/1560 objects degraded (2.179%) 2024-09-21T12:36:25.045 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:24 smithi148 ceph-mon[49387]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-09-21T12:36:25.045 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:24 smithi148 ceph-mon[49387]: Health check failed: Degraded data redundancy: 34/1560 objects degraded (2.179%), 5 pgs degraded (PG_DEGRADED) 2024-09-21T12:36:25.045 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:24 smithi148 podman[57207]: 2024-09-21 12:36:24.654892085 +0000 UTC m=+0.312702136 container create 18aa698f929a5f23502d39f0a8edb4873cf096c48c11b568e7550a82c3ba0e04 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6-activate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-21-0842-4417849, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-21T12:36:25.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:24 smithi078 ceph-mon[63866]: pgmap v266: 97 pgs: 2 active+undersized, 19 peering, 2 stale+active+clean, 5 active+undersized+degraded, 69 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.4 MiB/s wr, 388 op/s; 34/1560 objects degraded (2.179%) 2024-09-21T12:36:25.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:24 smithi078 ceph-mon[63866]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-09-21T12:36:25.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:24 smithi078 ceph-mon[63866]: Health check failed: Degraded data redundancy: 34/1560 objects degraded (2.179%), 5 pgs degraded (PG_DEGRADED) 2024-09-21T12:36:25.300 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:25 smithi148 podman[57207]: 2024-09-21 12:36:25.044200459 +0000 UTC m=+0.702010513 container init 18aa698f929a5f23502d39f0a8edb4873cf096c48c11b568e7550a82c3ba0e04 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6-activate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS) 2024-09-21T12:36:25.301 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:25 smithi148 podman[57207]: 2024-09-21 12:36:25.048235169 +0000 UTC m=+0.706045223 container start 18aa698f929a5f23502d39f0a8edb4873cf096c48c11b568e7550a82c3ba0e04 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6-activate, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-21T12:36:25.301 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:25 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6-activate[57222]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:36:25.301 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:25 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6-activate[57222]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:36:25.800 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:25 smithi148 podman[57207]: 2024-09-21 12:36:25.508546525 +0000 UTC m=+1.166356614 container attach 18aa698f929a5f23502d39f0a8edb4873cf096c48c11b568e7550a82c3ba0e04 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6-activate, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T12:36:25.800 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:25 smithi148 bash[57207]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:36:25.800 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:25 smithi148 bash[57207]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:36:25.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 246164 9.27 MB/sec execute 543 sec latency 69.127 ms 2024-09-21T12:36:26.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 246463 9.26 MB/sec execute 544 sec latency 47.439 ms 2024-09-21T12:36:27.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:26 smithi078 ceph-mon[63866]: pgmap v267: 97 pgs: 2 active+undersized, 19 peering, 1 stale+active+clean, 10 active+undersized+degraded, 65 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.2 MiB/s wr, 392 op/s; 70/1593 objects degraded (4.394%) 2024-09-21T12:36:27.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:26 smithi148 ceph-mon[49387]: pgmap v267: 97 pgs: 2 active+undersized, 19 peering, 1 stale+active+clean, 10 active+undersized+degraded, 65 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.2 MiB/s wr, 392 op/s; 70/1593 objects degraded (4.394%) 2024-09-21T12:36:27.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 246714 9.26 MB/sec execute 545 sec latency 45.308 ms 2024-09-21T12:36:27.997 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:27 smithi078 ceph-mon[63866]: pgmap v268: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 5.2 MiB/s wr, 534 op/s; 196/1674 objects degraded (11.708%) 2024-09-21T12:36:28.236 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:27 smithi148 ceph-mon[49387]: pgmap v268: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 5.2 MiB/s wr, 534 op/s; 196/1674 objects degraded (11.708%) 2024-09-21T12:36:28.550 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:28 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6-activate[57222]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T12:36:28.551 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:28 smithi148 bash[57207]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T12:36:28.551 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:28 smithi148 bash[57207]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:36:28.551 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:28 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6-activate[57222]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:36:28.551 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:28 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6-activate[57222]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:36:28.551 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:28 smithi148 bash[57207]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:36:28.551 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:28 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6-activate[57222]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-21T12:36:28.551 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:28 smithi148 bash[57207]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-21T12:36:28.551 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:28 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6-activate[57222]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-6 --no-mon-config 2024-09-21T12:36:28.551 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:28 smithi148 bash[57207]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-6 --no-mon-config 2024-09-21T12:36:28.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 246942 9.25 MB/sec execute 546 sec latency 43.210 ms 2024-09-21T12:36:28.896 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:28 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6-activate[57222]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-21T12:36:28.896 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:28 smithi148 bash[57207]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-21T12:36:28.896 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:28 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6-activate[57222]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-6/block 2024-09-21T12:36:28.896 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:28 smithi148 bash[57207]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-6/block 2024-09-21T12:36:28.896 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:28 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6-activate[57222]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T12:36:28.896 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:28 smithi148 bash[57207]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T12:36:28.896 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:28 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6-activate[57222]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-21T12:36:28.897 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:28 smithi148 bash[57207]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-21T12:36:28.897 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:28 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6-activate[57222]: --> ceph-volume lvm activate successful for osd ID: 6 2024-09-21T12:36:28.897 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:28 smithi148 bash[57207]: --> ceph-volume lvm activate successful for osd ID: 6 2024-09-21T12:36:28.897 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:28 smithi148 podman[57207]: 2024-09-21 12:36:28.637898414 +0000 UTC m=+4.295708482 container died 18aa698f929a5f23502d39f0a8edb4873cf096c48c11b568e7550a82c3ba0e04 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6-activate, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True) 2024-09-21T12:36:29.021 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:28 smithi078 ceph-mon[63866]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-09-21T12:36:29.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:28 smithi148 ceph-mon[49387]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-09-21T12:36:29.795 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:29 smithi148 podman[57437]: 2024-09-21 12:36:29.422997949 +0000 UTC m=+0.778249650 container remove 18aa698f929a5f23502d39f0a8edb4873cf096c48c11b568e7550a82c3ba0e04 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6-activate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-21T12:36:29.795 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:29 smithi148 podman[57497]: 2024-09-21 12:36:29.534402032 +0000 UTC m=+0.016955963 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:36:29.795 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:29 smithi148 podman[57497]: 2024-09-21 12:36:29.665613412 +0000 UTC m=+0.148167330 container create 17613684a6534c4b14e1248a618ddf07b88a8c1f0a73a75f11ef7202d6173094 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0842-4417849, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T12:36:29.831 INFO:teuthology.orchestra.run.smithi078.stdout: 5 247243 9.25 MB/sec execute 547 sec latency 60.842 ms 2024-09-21T12:36:30.050 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:29 smithi148 podman[57497]: 2024-09-21 12:36:29.927145596 +0000 UTC m=+0.409699524 container init 17613684a6534c4b14e1248a618ddf07b88a8c1f0a73a75f11ef7202d6173094 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-21T12:36:30.051 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:29 smithi148 podman[57497]: 2024-09-21 12:36:29.930630655 +0000 UTC m=+0.413184574 container start 17613684a6534c4b14e1248a618ddf07b88a8c1f0a73a75f11ef7202d6173094 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-21T12:36:30.051 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:29 smithi148 ceph-osd[57516]: -- 172.21.15.148:0/2643985401 <== mon.1 v2:172.21.15.148:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ad4f9741e0 con 0x55ad4eb17c00 2024-09-21T12:36:30.051 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:30 smithi148 bash[57497]: 17613684a6534c4b14e1248a618ddf07b88a8c1f0a73a75f11ef7202d6173094 2024-09-21T12:36:30.051 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:30 smithi148 systemd[1]: Started Ceph osd.6 for 6263deca-7813-11ef-baf6-efdc52797490. 2024-09-21T12:36:30.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 247529 9.24 MB/sec execute 548 sec latency 130.262 ms 2024-09-21T12:36:31.009 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:30 smithi148 ceph-mon[49387]: pgmap v269: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.3 MiB/s wr, 402 op/s; 200/1689 objects degraded (11.841%) 2024-09-21T12:36:31.009 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:31.009 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:36:31.009 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:30 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 200/1689 objects degraded (11.841%), 25 pgs degraded (PG_DEGRADED) 2024-09-21T12:36:31.009 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:31.009 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:31.009 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:36:31.073 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:30 smithi078 ceph-mon[63866]: pgmap v269: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.3 MiB/s wr, 402 op/s; 200/1689 objects degraded (11.841%) 2024-09-21T12:36:31.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:31.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:36:31.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:30 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 200/1689 objects degraded (11.841%), 25 pgs degraded (PG_DEGRADED) 2024-09-21T12:36:31.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:31.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:31.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:36:31.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 247822 9.24 MB/sec execute 549 sec latency 41.907 ms 2024-09-21T12:36:32.050 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:31 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:36:31.818+0000 7f560df6a640 -1 Falling back to public interface 2024-09-21T12:36:32.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 248037 9.23 MB/sec execute 550 sec latency 43.189 ms 2024-09-21T12:36:33.118 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:32 smithi078 ceph-mon[63866]: pgmap v270: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 4.2 MiB/s wr, 369 op/s; 197/1635 objects degraded (12.049%) 2024-09-21T12:36:33.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:32 smithi148 ceph-mon[49387]: pgmap v270: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 4.2 MiB/s wr, 369 op/s; 197/1635 objects degraded (12.049%) 2024-09-21T12:36:33.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 248320 9.23 MB/sec execute 551 sec latency 64.874 ms 2024-09-21T12:36:34.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 248640 9.23 MB/sec execute 552 sec latency 45.011 ms 2024-09-21T12:36:34.915 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:34 smithi078 ceph-mon[63866]: pgmap v271: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 328 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.2 MiB/s wr, 451 op/s; 184/1557 objects degraded (11.818%) 2024-09-21T12:36:34.915 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:34 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:34.915 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:34 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:34.915 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:34 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:34.915 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:34 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:35.166 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:34 smithi148 ceph-mon[49387]: pgmap v271: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 328 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.2 MiB/s wr, 451 op/s; 184/1557 objects degraded (11.818%) 2024-09-21T12:36:35.166 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:34 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:35.166 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:34 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:35.166 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:34 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:35.166 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:34 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:35.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 248867 9.23 MB/sec execute 553 sec latency 66.026 ms 2024-09-21T12:36:35.942 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:35 smithi078 ceph-mon[63866]: pgmap v272: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.4 MiB/s wr, 438 op/s; 184/1566 objects degraded (11.750%) 2024-09-21T12:36:35.943 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:35 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 184/1566 objects degraded (11.750%), 25 pgs degraded (PG_DEGRADED) 2024-09-21T12:36:36.246 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:35 smithi148 ceph-mon[49387]: pgmap v272: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.4 MiB/s wr, 438 op/s; 184/1566 objects degraded (11.750%) 2024-09-21T12:36:36.246 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:35 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 184/1566 objects degraded (11.750%), 25 pgs degraded (PG_DEGRADED) 2024-09-21T12:36:36.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 249095 9.22 MB/sec execute 554 sec latency 92.285 ms 2024-09-21T12:36:37.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 249322 9.23 MB/sec execute 555 sec latency 84.824 ms 2024-09-21T12:36:38.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:38 smithi078 ceph-mon[63866]: pgmap v273: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.8 MiB/s wr, 451 op/s; 186/1584 objects degraded (11.742%) 2024-09-21T12:36:38.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:38 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:38.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:38 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:38.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:38 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:36:38.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:38 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:36:38.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:38 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:38.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:38 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:38.616 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:38 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:36:38.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:38 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:36:38.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:38 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:36:38.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:38 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:36:38.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:38 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-21T12:36:38.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 249649 9.22 MB/sec execute 556 sec latency 65.591 ms 2024-09-21T12:36:39.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:38 smithi148 ceph-mon[49387]: pgmap v273: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 329 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.8 MiB/s wr, 451 op/s; 186/1584 objects degraded (11.742%) 2024-09-21T12:36:39.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:38 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:39.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:38 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:39.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:38 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:36:39.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:38 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:36:39.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:38 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:39.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:38 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:39.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:38 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:36:39.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:38 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:36:39.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:38 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:36:39.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:38 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:36:39.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:38 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-21T12:36:39.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:39 smithi078 ceph-mon[63866]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T12:36:39.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:39 smithi078 ceph-mon[63866]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-21T12:36:39.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:39 smithi148 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T12:36:39.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:39 smithi148 ceph-mon[49387]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-21T12:36:39.801 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:39 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:36:39.506+0000 7f560df6a640 -1 osd.6 0 read_superblock omap replica is missing. 2024-09-21T12:36:39.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 249997 9.22 MB/sec execute 557 sec latency 42.091 ms 2024-09-21T12:36:40.687 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:40 smithi078 ceph-mon[63866]: pgmap v274: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 3.6 MiB/s wr, 366 op/s; 189/1590 objects degraded (11.887%) 2024-09-21T12:36:40.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 250222 9.21 MB/sec execute 558 sec latency 53.465 ms 2024-09-21T12:36:41.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:40 smithi148 ceph-mon[49387]: pgmap v274: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 3.6 MiB/s wr, 366 op/s; 189/1590 objects degraded (11.887%) 2024-09-21T12:36:41.050 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:40 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:36:40.675+0000 7f560df6a640 -1 osd.6 141 log_to_monitors true 2024-09-21T12:36:41.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:41 smithi078 ceph-mon[63866]: from='osd.6 [v2:172.21.15.148:6816/623540096,v1:172.21.15.148:6817/623540096]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-21T12:36:41.732 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:41 smithi078 ceph-mon[63866]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-21T12:36:41.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 250460 9.21 MB/sec execute 559 sec latency 87.747 ms 2024-09-21T12:36:42.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:41 smithi148 ceph-mon[49387]: from='osd.6 [v2:172.21.15.148:6816/623540096,v1:172.21.15.148:6817/623540096]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-21T12:36:42.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:41 smithi148 ceph-mon[49387]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-21T12:36:42.051 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:36:41 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:36:41.655+0000 7f5605d14640 -1 osd.6 141 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T12:36:42.753 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:42 smithi078 ceph-mon[63866]: pgmap v275: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.2 MiB/s wr, 394 op/s; 201/1650 objects degraded (12.182%) 2024-09-21T12:36:42.753 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:42 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 201/1650 objects degraded (12.182%), 25 pgs degraded (PG_DEGRADED) 2024-09-21T12:36:42.753 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:42 smithi078 ceph-mon[63866]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T12:36:42.753 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:42 smithi078 ceph-mon[63866]: osdmap e144: 8 total, 7 up, 8 in 2024-09-21T12:36:42.753 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:42 smithi078 ceph-mon[63866]: from='osd.6 [v2:172.21.15.148:6816/623540096,v1:172.21.15.148:6817/623540096]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi148", "root=default"]} : dispatch 2024-09-21T12:36:42.753 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:42 smithi078 ceph-mon[63866]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi148", "root=default"]} : dispatch 2024-09-21T12:36:42.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 250740 9.20 MB/sec execute 560 sec latency 143.328 ms 2024-09-21T12:36:43.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:42 smithi148 ceph-mon[49387]: pgmap v275: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.2 MiB/s wr, 394 op/s; 201/1650 objects degraded (12.182%) 2024-09-21T12:36:43.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:42 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 201/1650 objects degraded (12.182%), 25 pgs degraded (PG_DEGRADED) 2024-09-21T12:36:43.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:42 smithi148 ceph-mon[49387]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T12:36:43.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:42 smithi148 ceph-mon[49387]: osdmap e144: 8 total, 7 up, 8 in 2024-09-21T12:36:43.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:42 smithi148 ceph-mon[49387]: from='osd.6 [v2:172.21.15.148:6816/623540096,v1:172.21.15.148:6817/623540096]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi148", "root=default"]} : dispatch 2024-09-21T12:36:43.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:42 smithi148 ceph-mon[49387]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi148", "root=default"]} : dispatch 2024-09-21T12:36:43.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 250859 9.19 MB/sec execute 561 sec latency 1143.396 ms 2024-09-21T12:36:44.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:43 smithi148 ceph-mon[49387]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T12:36:44.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:43 smithi148 ceph-mon[49387]: osd.6 [v2:172.21.15.148:6816/623540096,v1:172.21.15.148:6817/623540096] boot 2024-09-21T12:36:44.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:43 smithi148 ceph-mon[49387]: osdmap e145: 8 total, 8 up, 8 in 2024-09-21T12:36:44.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:43 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-21T12:36:44.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:43 smithi078 ceph-mon[63866]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T12:36:44.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:43 smithi078 ceph-mon[63866]: osd.6 [v2:172.21.15.148:6816/623540096,v1:172.21.15.148:6817/623540096] boot 2024-09-21T12:36:44.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:43 smithi078 ceph-mon[63866]: osdmap e145: 8 total, 8 up, 8 in 2024-09-21T12:36:44.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:43 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-21T12:36:44.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 250923 9.18 MB/sec execute 562 sec latency 1996.897 ms 2024-09-21T12:36:44.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:44 smithi078 ceph-mon[63866]: pgmap v278: 97 pgs: 5 peering, 9 active+undersized, 21 active+undersized+degraded, 62 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 4.7 MiB/s wr, 445 op/s; 164/1632 objects degraded (10.049%) 2024-09-21T12:36:44.902 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:44 smithi078 ceph-mon[63866]: osdmap e146: 8 total, 8 up, 8 in 2024-09-21T12:36:45.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:44 smithi148 ceph-mon[49387]: pgmap v278: 97 pgs: 5 peering, 9 active+undersized, 21 active+undersized+degraded, 62 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 4.7 MiB/s wr, 445 op/s; 164/1632 objects degraded (10.049%) 2024-09-21T12:36:45.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:44 smithi148 ceph-mon[49387]: osdmap e146: 8 total, 8 up, 8 in 2024-09-21T12:36:45.658 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:45 smithi078 ceph-mon[63866]: osdmap e147: 8 total, 8 up, 8 in 2024-09-21T12:36:45.659 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:45.659 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:36:45.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 251215 9.17 MB/sec execute 563 sec latency 181.555 ms 2024-09-21T12:36:45.943 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:45 smithi148 ceph-mon[49387]: osdmap e147: 8 total, 8 up, 8 in 2024-09-21T12:36:45.943 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:45.944 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:36:46.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 251267 9.16 MB/sec execute 564 sec latency 1040.923 ms 2024-09-21T12:36:46.921 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:46 smithi078 ceph-mon[63866]: pgmap v281: 97 pgs: 1 active+undersized+remapped, 5 peering, 9 active+undersized, 17 active+undersized+degraded, 65 active+clean; 336 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s wr, 296 op/s; 130/1584 objects degraded (8.207%); 0 B/s, 8 objects/s recovering 2024-09-21T12:36:46.922 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:46 smithi078 ceph-mon[63866]: osdmap e148: 8 total, 8 up, 8 in 2024-09-21T12:36:47.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:46 smithi148 ceph-mon[49387]: pgmap v281: 97 pgs: 1 active+undersized+remapped, 5 peering, 9 active+undersized, 17 active+undersized+degraded, 65 active+clean; 336 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s wr, 296 op/s; 130/1584 objects degraded (8.207%); 0 B/s, 8 objects/s recovering 2024-09-21T12:36:47.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:46 smithi148 ceph-mon[49387]: osdmap e148: 8 total, 8 up, 8 in 2024-09-21T12:36:47.696 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:47 smithi078 ceph-mon[63866]: osdmap e149: 8 total, 8 up, 8 in 2024-09-21T12:36:47.697 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:47 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 130/1584 objects degraded (8.207%), 17 pgs degraded (PG_DEGRADED) 2024-09-21T12:36:47.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 251573 9.16 MB/sec execute 565 sec latency 36.729 ms 2024-09-21T12:36:48.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:47 smithi148 ceph-mon[49387]: osdmap e149: 8 total, 8 up, 8 in 2024-09-21T12:36:48.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:47 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 130/1584 objects degraded (8.207%), 17 pgs degraded (PG_DEGRADED) 2024-09-21T12:36:48.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:48 smithi078 ceph-mon[63866]: pgmap v284: 97 pgs: 1 active+undersized+remapped, 6 peering, 90 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 1.8 MiB/s wr, 200 op/s; 2.7 MiB/s, 54 keys/s, 49 objects/s recovering 2024-09-21T12:36:48.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:48 smithi078 ceph-mon[63866]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 130/1584 objects degraded (8.207%), 17 pgs degraded) 2024-09-21T12:36:48.729 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:48 smithi078 ceph-mon[63866]: Cluster is now healthy 2024-09-21T12:36:48.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 251843 9.15 MB/sec execute 566 sec latency 50.851 ms 2024-09-21T12:36:49.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:48 smithi148 ceph-mon[49387]: pgmap v284: 97 pgs: 1 active+undersized+remapped, 6 peering, 90 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 1.8 MiB/s wr, 200 op/s; 2.7 MiB/s, 54 keys/s, 49 objects/s recovering 2024-09-21T12:36:49.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:48 smithi148 ceph-mon[49387]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 130/1584 objects degraded (8.207%), 17 pgs degraded) 2024-09-21T12:36:49.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:48 smithi148 ceph-mon[49387]: Cluster is now healthy 2024-09-21T12:36:49.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 252346 9.17 MB/sec execute 567 sec latency 36.307 ms 2024-09-21T12:36:50.759 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:50 smithi078 ceph-mon[63866]: pgmap v285: 97 pgs: 1 active+undersized+remapped, 1 peering, 95 active+clean; 332 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 1.8 MiB/s wr, 271 op/s; 1.9 MiB/s, 36 keys/s, 40 objects/s recovering 2024-09-21T12:36:50.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 252765 9.18 MB/sec execute 568 sec latency 39.654 ms 2024-09-21T12:36:51.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:50 smithi148 ceph-mon[49387]: pgmap v285: 97 pgs: 1 active+undersized+remapped, 1 peering, 95 active+clean; 332 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 1.8 MiB/s wr, 271 op/s; 1.9 MiB/s, 36 keys/s, 40 objects/s recovering 2024-09-21T12:36:51.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 253101 9.18 MB/sec execute 569 sec latency 59.225 ms 2024-09-21T12:36:52.784 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:52 smithi078 ceph-mon[63866]: pgmap v286: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 2.8 MiB/s wr, 284 op/s; 1.6 MiB/s, 31 keys/s, 34 objects/s recovering 2024-09-21T12:36:52.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 253460 9.17 MB/sec execute 570 sec latency 66.141 ms 2024-09-21T12:36:53.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:52 smithi148 ceph-mon[49387]: pgmap v286: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 2.8 MiB/s wr, 284 op/s; 1.6 MiB/s, 31 keys/s, 34 objects/s recovering 2024-09-21T12:36:53.188 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-21T12:36:53.791 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:53 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-21T12:36:53.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 253688 9.16 MB/sec execute 571 sec latency 45.708 ms 2024-09-21T12:36:54.032 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:53 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-21T12:36:54.267 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T12:36:54.267 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (9m) 2m ago 18m 20.0M - 0.27.0 11f11916f8cd e6e67ab07dec 2024-09-21T12:36:54.267 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (5m) 2m ago 18m 6895k - 19.3.0-5104-g44178494 b415816db8b4 46a10ea0b111 2024-09-21T12:36:54.267 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi148 smithi148 running (5m) 20s ago 17m 6895k - 19.3.0-5104-g44178494 b415816db8b4 23deb0615621 2024-09-21T12:36:54.267 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (8m) 2m ago 17m 53.3M - dad864ee21e9 2a140c36091b 2024-09-21T12:36:54.267 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.ytpzwh smithi078 running (12m) 2m ago 12m 38.4M - 17.2.0 e1d6a67b021e 5e9b109502a2 2024-09-21T12:36:54.267 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi148.uukezr smithi148 running (12m) 20s ago 12m 14.7M - 17.2.0 e1d6a67b021e b7e8aa0f3a6c 2024-09-21T12:36:54.267 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.ntwyan smithi078 *:8443,9283,8765 running (8m) 2m ago 19m 519M - 19.3.0-5104-g44178494 b415816db8b4 a61baedebf43 2024-09-21T12:36:54.268 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi148.oyiqmx smithi148 *:8443,9283,8765 running (7m) 20s ago 17m 457M - 19.3.0-5104-g44178494 b415816db8b4 0255535c6fdb 2024-09-21T12:36:54.268 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (7m) 2m ago 19m 55.9M 2048M 19.3.0-5104-g44178494 b415816db8b4 22b3097efbd8 2024-09-21T12:36:54.268 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi148 smithi148 running (6m) 20s ago 17m 44.8M 2048M 19.3.0-5104-g44178494 b415816db8b4 df8b08509136 2024-09-21T12:36:54.268 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.ydbhru smithi078 *:2049 running (11m) 2m ago 11m 279M - 4.0 e1d6a67b021e 7e5b0274ac38 2024-09-21T12:36:54.268 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi148.einuzt smithi148 *:2049 running (11m) 20s ago 11m 87.3M - 4.0 e1d6a67b021e 76a8b0eabede 2024-09-21T12:36:54.268 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (8m) 2m ago 17m 11.0M - 1.7.0 72c9c2088986 301adb4e9e22 2024-09-21T12:36:54.268 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi148 smithi148 *:9100 running (8m) 20s ago 17m 12.2M - 1.7.0 72c9c2088986 0cceff4bf342 2024-09-21T12:36:54.268 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (5m) 2m ago 16m 345M 1936M 19.3.0-5104-g44178494 b415816db8b4 7374add97534 2024-09-21T12:36:54.268 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (4m) 2m ago 16m 276M 1936M 19.3.0-5104-g44178494 b415816db8b4 14655a8da4e5 2024-09-21T12:36:54.268 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (3m) 2m ago 15m 219M 1936M 19.3.0-5104-g44178494 b415816db8b4 07ceda32f383 2024-09-21T12:36:54.268 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (2m) 2m ago 15m 11.4M 1936M 19.3.0-5104-g44178494 b415816db8b4 ead87e4306ff 2024-09-21T12:36:54.268 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi148 running (103s) 20s ago 14m 296M 2704M 19.3.0-5104-g44178494 b415816db8b4 10ec7d6b476e 2024-09-21T12:36:54.268 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi148 running (64s) 20s ago 14m 258M 2704M 19.3.0-5104-g44178494 b415816db8b4 06ac02700ea1 2024-09-21T12:36:54.268 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi148 running (24s) 20s ago 14m 12.1M 2704M 19.3.0-5104-g44178494 b415816db8b4 17613684a653 2024-09-21T12:36:54.269 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi148 running (13m) 20s ago 13m 754M 2704M 17.2.0 e1d6a67b021e f9e4db9d35fa 2024-09-21T12:36:54.269 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (7m) 2m ago 17m 55.8M - 2.51.0 1d3b7f56885b face2d4cdd1b 2024-09-21T12:36:54.802 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:54 smithi078 ceph-mon[63866]: from='client.34426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:36:54.803 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:54 smithi078 ceph-mon[63866]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T12:36:54.803 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:54 smithi078 ceph-mon[63866]: Upgrade: osd.7 is safe to restart 2024-09-21T12:36:54.803 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:54 smithi078 ceph-mon[63866]: pgmap v287: 97 pgs: 97 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.4 MiB/s wr, 454 op/s; 1.4 MiB/s, 27 keys/s, 25 objects/s recovering 2024-09-21T12:36:54.803 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:54 smithi078 ceph-mon[63866]: from='client.34430 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:36:54.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 253956 9.16 MB/sec execute 572 sec latency 51.369 ms 2024-09-21T12:36:54.848 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:36:54.848 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-21T12:36:54.848 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:36:54.848 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:36:54.848 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-21T12:36:54.848 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:36:54.848 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:36:54.848 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-21T12:36:54.848 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-21T12:36:54.848 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 7 2024-09-21T12:36:54.848 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:36:54.849 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-21T12:36:54.849 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:36:54.849 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:36:54.849 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-21T12:36:54.849 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-21T12:36:54.849 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 11 2024-09-21T12:36:54.849 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T12:36:54.849 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:36:55.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:54 smithi148 ceph-mon[49387]: from='client.34426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:36:55.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:54 smithi148 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T12:36:55.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:54 smithi148 ceph-mon[49387]: Upgrade: osd.7 is safe to restart 2024-09-21T12:36:55.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:54 smithi148 ceph-mon[49387]: pgmap v287: 97 pgs: 97 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.4 MiB/s wr, 454 op/s; 1.4 MiB/s, 27 keys/s, 25 objects/s recovering 2024-09-21T12:36:55.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:54 smithi148 ceph-mon[49387]: from='client.34430 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:36:55.338 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:36:55.338 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5", 2024-09-21T12:36:55.338 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-21T12:36:55.339 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T12:36:55.339 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-21T12:36:55.339 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-21T12:36:55.339 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-09-21T12:36:55.339 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr" 2024-09-21T12:36:55.339 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-21T12:36:55.339 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "13/23 daemons upgraded", 2024-09-21T12:36:55.339 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T12:36:55.340 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-21T12:36:55.340 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:36:55.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 254278 9.16 MB/sec execute 573 sec latency 49.464 ms 2024-09-21T12:36:55.871 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-21T12:36:56.025 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:55 smithi148 ceph-mon[49387]: from='client.44331 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:36:56.025 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:55 smithi148 ceph-mon[49387]: Upgrade: Updating osd.7 2024-09-21T12:36:56.025 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:55 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:56.025 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:55 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T12:36:56.025 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:55 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:36:56.026 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:55 smithi148 ceph-mon[49387]: Deploying daemon osd.7 on smithi148 2024-09-21T12:36:56.026 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:55 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/3003436041' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:36:56.093 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:55 smithi078 ceph-mon[63866]: from='client.44331 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:36:56.094 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:55 smithi078 ceph-mon[63866]: Upgrade: Updating osd.7 2024-09-21T12:36:56.094 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:55 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:36:56.094 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:55 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T12:36:56.094 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:55 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:36:56.094 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:55 smithi078 ceph-mon[63866]: Deploying daemon osd.7 on smithi148 2024-09-21T12:36:56.094 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:55 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/3003436041' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:36:56.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 254618 9.16 MB/sec execute 574 sec latency 65.175 ms 2024-09-21T12:36:57.111 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:56 smithi078 ceph-mon[63866]: from='client.34442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:36:57.112 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:56 smithi078 ceph-mon[63866]: pgmap v288: 97 pgs: 97 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 3.5 MiB/s wr, 367 op/s; 1.1 MiB/s, 21 keys/s, 20 objects/s recovering 2024-09-21T12:36:57.112 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:56 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/182944282' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:36:57.139 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:56 smithi148 ceph-mon[49387]: from='client.34442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:36:57.139 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:56 smithi148 ceph-mon[49387]: pgmap v288: 97 pgs: 97 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 3.5 MiB/s wr, 367 op/s; 1.1 MiB/s, 21 keys/s, 20 objects/s recovering 2024-09-21T12:36:57.139 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:56 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/182944282' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:36:57.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 254893 9.15 MB/sec execute 575 sec latency 61.651 ms 2024-09-21T12:36:58.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 255225 9.15 MB/sec execute 576 sec latency 57.706 ms 2024-09-21T12:36:58.920 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:58 smithi078 ceph-mon[63866]: pgmap v289: 97 pgs: 97 active+clean; 341 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 4.8 MiB/s wr, 500 op/s; 996 KiB/s, 16 objects/s recovering 2024-09-21T12:36:59.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:58 smithi148 ceph-mon[49387]: pgmap v289: 97 pgs: 97 active+clean; 341 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 4.8 MiB/s wr, 500 op/s; 996 KiB/s, 16 objects/s recovering 2024-09-21T12:36:59.051 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:36:58 smithi148 systemd[1]: Stopping Ceph osd.7 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:36:59.550 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:36:59 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[38978]: 2024-09-21T12:36:59.196+0000 7ff7e0043700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T12:36:59.550 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:36:59 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[38978]: 2024-09-21T12:36:59.196+0000 7ff7e0043700 -1 osd.7 149 *** Got signal Terminated *** 2024-09-21T12:36:59.551 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:36:59 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[38978]: 2024-09-21T12:36:59.196+0000 7ff7e0043700 -1 osd.7 149 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T12:36:59.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 255520 9.15 MB/sec execute 577 sec latency 64.976 ms 2024-09-21T12:37:00.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:36:59 smithi078 ceph-mon[63866]: osd.7 marked itself down and dead 2024-09-21T12:37:00.229 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:36:59 smithi148 ceph-mon[49387]: osd.7 marked itself down and dead 2024-09-21T12:37:00.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:00 smithi148 podman[59091]: 2024-09-21 12:37:00.538383202 +0000 UTC m=+1.444618103 container died f9e4db9d35fa7ea3e5102013474fcd69392b9cae97e5a08a45026b1b716c246b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, io.openshift.expose-services=, architecture=x86_64, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, RELEASE=HEAD, version=8, CEPH_POINT_RELEASE=-17.2.0) 2024-09-21T12:37:00.833 INFO:teuthology.orchestra.run.smithi078.stdout: 5 255523 9.13 MB/sec execute 578 sec latency 992.126 ms 2024-09-21T12:37:01.124 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:00 smithi078 ceph-mon[63866]: pgmap v290: 97 pgs: 97 active+clean; 336 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.2 MiB/s wr, 445 op/s; 19 KiB/s, 3 objects/s recovering 2024-09-21T12:37:01.124 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:00 smithi078 ceph-mon[63866]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T12:37:01.124 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:00 smithi078 ceph-mon[63866]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-21T12:37:01.124 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:00 smithi078 ceph-mon[63866]: osdmap e150: 8 total, 7 up, 8 in 2024-09-21T12:37:01.124 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:01.124 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:37:01.124 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:01.199 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:00 smithi148 podman[59091]: 2024-09-21 12:37:00.912729219 +0000 UTC m=+1.818964130 container cleanup f9e4db9d35fa7ea3e5102013474fcd69392b9cae97e5a08a45026b1b716c246b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, name=centos-stream, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, io.openshift.expose-services=, GIT_BRANCH=HEAD) 2024-09-21T12:37:01.199 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:00 smithi148 bash[59091]: ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7 2024-09-21T12:37:01.199 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:00 smithi148 ceph-mon[49387]: pgmap v290: 97 pgs: 97 active+clean; 336 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.2 MiB/s wr, 445 op/s; 19 KiB/s, 3 objects/s recovering 2024-09-21T12:37:01.199 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:00 smithi148 ceph-mon[49387]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T12:37:01.199 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:00 smithi148 ceph-mon[49387]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-21T12:37:01.199 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:00 smithi148 ceph-mon[49387]: osdmap e150: 8 total, 7 up, 8 in 2024-09-21T12:37:01.199 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:01.200 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:37:01.200 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:01.550 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:01 smithi148 podman[59118]: 2024-09-21 12:37:01.19778997 +0000 UTC m=+0.656133038 container remove f9e4db9d35fa7ea3e5102013474fcd69392b9cae97e5a08a45026b1b716c246b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7, GIT_BRANCH=HEAD, io.openshift.expose-services=, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, version=8, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, ceph=True, CEPH_POINT_RELEASE=-17.2.0, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, distribution-scope=public, GIT_CLEAN=True, io.buildah.version=1.19.8, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-21T12:37:01.827 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:01 smithi148 podman[59205]: 2024-09-21 12:37:01.469585785 +0000 UTC m=+0.018139667 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:37:01.827 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:01 smithi148 podman[59205]: 2024-09-21 12:37:01.590387564 +0000 UTC m=+0.138941449 container create 1a6704616755d0f45e5005d6bd20e6be83173cccbc839d87f4221246b2630ba1 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-21T12:37:01.832 INFO:teuthology.orchestra.run.smithi078.stdout: 5 255757 9.13 MB/sec execute 579 sec latency 1037.493 ms 2024-09-21T12:37:01.915 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:01 smithi078 ceph-mon[63866]: osdmap e151: 8 total, 7 up, 8 in 2024-09-21T12:37:02.294 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:01 smithi148 ceph-mon[49387]: osdmap e151: 8 total, 7 up, 8 in 2024-09-21T12:37:02.295 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:01 smithi148 podman[59205]: 2024-09-21 12:37:01.826626443 +0000 UTC m=+0.375180332 container init 1a6704616755d0f45e5005d6bd20e6be83173cccbc839d87f4221246b2630ba1 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T12:37:02.295 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:01 smithi148 podman[59205]: 2024-09-21 12:37:01.830152075 +0000 UTC m=+0.378705963 container start 1a6704616755d0f45e5005d6bd20e6be83173cccbc839d87f4221246b2630ba1 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2) 2024-09-21T12:37:02.295 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:02 smithi148 podman[59205]: 2024-09-21 12:37:02.293877742 +0000 UTC m=+0.842431634 container attach 1a6704616755d0f45e5005d6bd20e6be83173cccbc839d87f4221246b2630ba1 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True) 2024-09-21T12:37:02.295 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:02 smithi148 podman[59205]: 2024-09-21 12:37:02.294231344 +0000 UTC m=+0.842785234 container died 1a6704616755d0f45e5005d6bd20e6be83173cccbc839d87f4221246b2630ba1 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7-deactivate, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-21T12:37:02.833 INFO:teuthology.orchestra.run.smithi078.stdout: 5 256017 9.13 MB/sec execute 580 sec latency 63.706 ms 2024-09-21T12:37:02.922 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:02 smithi078 ceph-mon[63866]: pgmap v293: 97 pgs: 3 active+undersized, 4 peering, 9 stale+active+clean, 6 active+undersized+degraded, 75 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s wr, 325 op/s; 48/1569 objects degraded (3.059%) 2024-09-21T12:37:02.923 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:02 smithi078 ceph-mon[63866]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T12:37:02.923 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:02 smithi078 ceph-mon[63866]: Health check failed: Degraded data redundancy: 48/1569 objects degraded (3.059%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T12:37:03.165 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:02 smithi148 ceph-mon[49387]: pgmap v293: 97 pgs: 3 active+undersized, 4 peering, 9 stale+active+clean, 6 active+undersized+degraded, 75 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s wr, 325 op/s; 48/1569 objects degraded (3.059%) 2024-09-21T12:37:03.166 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:02 smithi148 ceph-mon[49387]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T12:37:03.166 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:02 smithi148 ceph-mon[49387]: Health check failed: Degraded data redundancy: 48/1569 objects degraded (3.059%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T12:37:03.166 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:02 smithi148 podman[59229]: 2024-09-21 12:37:02.924097853 +0000 UTC m=+0.868207650 container remove 1a6704616755d0f45e5005d6bd20e6be83173cccbc839d87f4221246b2630ba1 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2) 2024-09-21T12:37:03.166 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:02 smithi148 systemd[1]: ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.7.service: Deactivated successfully. 2024-09-21T12:37:03.166 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:02 smithi148 systemd[1]: Stopped Ceph osd.7 for 6263deca-7813-11ef-baf6-efdc52797490. 2024-09-21T12:37:03.166 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:02 smithi148 systemd[1]: ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.7.service: Consumed 6min 8.679s CPU time. 2024-09-21T12:37:03.418 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:03 smithi148 systemd[1]: Starting Ceph osd.7 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:37:03.418 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:03 smithi148 podman[59328]: 2024-09-21 12:37:03.288355677 +0000 UTC m=+0.017685678 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:37:03.670 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:03 smithi148 podman[59328]: 2024-09-21 12:37:03.417173207 +0000 UTC m=+0.146503263 container create a79cdc3b3cd6faeba87e0ff1b90601cff97168ed39ae9ad465580a69290effca (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7-activate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0842-4417849, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-21T12:37:03.833 INFO:teuthology.orchestra.run.smithi078.stdout: 5 256247 9.13 MB/sec execute 581 sec latency 61.476 ms 2024-09-21T12:37:03.926 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:03 smithi148 podman[59328]: 2024-09-21 12:37:03.669886374 +0000 UTC m=+0.399216370 container init a79cdc3b3cd6faeba87e0ff1b90601cff97168ed39ae9ad465580a69290effca (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0842-4417849, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-21T12:37:03.926 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:03 smithi148 podman[59328]: 2024-09-21 12:37:03.673742648 +0000 UTC m=+0.403072634 container start a79cdc3b3cd6faeba87e0ff1b90601cff97168ed39ae9ad465580a69290effca (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7-activate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-21T12:37:03.926 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:03 smithi148 podman[59328]: 2024-09-21 12:37:03.769989857 +0000 UTC m=+0.499319853 container attach a79cdc3b3cd6faeba87e0ff1b90601cff97168ed39ae9ad465580a69290effca (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7-activate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-21-0842-4417849, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T12:37:03.926 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:03 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7-activate[59342]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:37:03.926 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:03 smithi148 bash[59328]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:37:03.926 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:03 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7-activate[59342]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:37:03.927 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:03 smithi148 bash[59328]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:37:03.979 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:03 smithi078 ceph-mon[63866]: pgmap v294: 97 pgs: 7 active+undersized, 4 peering, 31 active+undersized+degraded, 55 active+clean; 336 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.0 MiB/s wr, 537 op/s; 230/1578 objects degraded (14.575%) 2024-09-21T12:37:04.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:03 smithi148 ceph-mon[49387]: pgmap v294: 97 pgs: 7 active+undersized, 4 peering, 31 active+undersized+degraded, 55 active+clean; 336 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.0 MiB/s wr, 537 op/s; 230/1578 objects degraded (14.575%) 2024-09-21T12:37:04.833 INFO:teuthology.orchestra.run.smithi078.stdout: 5 256480 9.13 MB/sec execute 582 sec latency 51.175 ms 2024-09-21T12:37:05.833 INFO:teuthology.orchestra.run.smithi078.stdout: 5 256753 9.12 MB/sec execute 583 sec latency 70.029 ms 2024-09-21T12:37:06.780 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:06 smithi078 ceph-mon[63866]: pgmap v295: 97 pgs: 7 active+undersized, 4 peering, 31 active+undersized+degraded, 55 active+clean; 336 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s wr, 249 op/s; 230/1578 objects degraded (14.575%) 2024-09-21T12:37:06.833 INFO:teuthology.orchestra.run.smithi078.stdout: 5 257009 9.12 MB/sec execute 584 sec latency 55.959 ms 2024-09-21T12:37:07.004 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:06 smithi148 ceph-mon[49387]: pgmap v295: 97 pgs: 7 active+undersized, 4 peering, 31 active+undersized+degraded, 55 active+clean; 336 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s wr, 249 op/s; 230/1578 objects degraded (14.575%) 2024-09-21T12:37:07.004 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:06 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7-activate[59342]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T12:37:07.004 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:06 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7-activate[59342]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:37:07.004 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:06 smithi148 bash[59328]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-21T12:37:07.004 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:06 smithi148 bash[59328]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:37:07.004 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:06 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7-activate[59342]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:37:07.004 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:06 smithi148 bash[59328]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T12:37:07.004 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:06 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7-activate[59342]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-21T12:37:07.004 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:06 smithi148 bash[59328]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-21T12:37:07.005 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:06 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7-activate[59342]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-7 --no-mon-config 2024-09-21T12:37:07.005 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:06 smithi148 bash[59328]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-7 --no-mon-config 2024-09-21T12:37:07.300 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:07 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7-activate[59342]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-21T12:37:07.300 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:07 smithi148 bash[59328]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-21T12:37:07.301 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:07 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7-activate[59342]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-7/block 2024-09-21T12:37:07.301 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:07 smithi148 bash[59328]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-7/block 2024-09-21T12:37:07.301 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:07 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7-activate[59342]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T12:37:07.301 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:07 smithi148 bash[59328]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T12:37:07.301 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:07 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7-activate[59342]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-21T12:37:07.301 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:07 smithi148 bash[59328]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-21T12:37:07.301 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:07 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7-activate[59342]: --> ceph-volume lvm activate successful for osd ID: 7 2024-09-21T12:37:07.301 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:07 smithi148 bash[59328]: --> ceph-volume lvm activate successful for osd ID: 7 2024-09-21T12:37:07.301 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:07 smithi148 podman[59328]: 2024-09-21 12:37:07.124560213 +0000 UTC m=+3.853890208 container died a79cdc3b3cd6faeba87e0ff1b90601cff97168ed39ae9ad465580a69290effca (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7-activate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-21-0842-4417849, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=) 2024-09-21T12:37:07.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:07 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 230/1578 objects degraded (14.575%), 31 pgs degraded (PG_DEGRADED) 2024-09-21T12:37:07.816 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:07 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 230/1578 objects degraded (14.575%), 31 pgs degraded (PG_DEGRADED) 2024-09-21T12:37:07.833 INFO:teuthology.orchestra.run.smithi078.stdout: 5 257165 9.11 MB/sec execute 585 sec latency 229.701 ms 2024-09-21T12:37:08.095 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:07 smithi148 podman[59546]: 2024-09-21 12:37:07.801673193 +0000 UTC m=+0.667938280 container remove a79cdc3b3cd6faeba87e0ff1b90601cff97168ed39ae9ad465580a69290effca (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7-activate, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-21T12:37:08.096 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:08 smithi148 podman[59603]: 2024-09-21 12:37:07.950096042 +0000 UTC m=+0.029979070 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:37:08.096 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:08 smithi148 podman[59603]: 2024-09-21 12:37:08.094651716 +0000 UTC m=+0.174534726 container create 2e6a6f842c47f317e3b177da026caf78ade2f25a8f4925cf9aa84f2f7d13f7c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD) 2024-09-21T12:37:08.367 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:08 smithi148 podman[59603]: 2024-09-21 12:37:08.339241974 +0000 UTC m=+0.419124998 container init 2e6a6f842c47f317e3b177da026caf78ade2f25a8f4925cf9aa84f2f7d13f7c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T12:37:08.367 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:08 smithi148 podman[59603]: 2024-09-21 12:37:08.343463635 +0000 UTC m=+0.423346636 container start 2e6a6f842c47f317e3b177da026caf78ade2f25a8f4925cf9aa84f2f7d13f7c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731) 2024-09-21T12:37:08.630 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:08 smithi148 ceph-osd[59620]: -- 172.21.15.148:0/1567610049 <== mon.1 v2:172.21.15.148:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56269fc481e0 con 0x56269fbda000 2024-09-21T12:37:08.631 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:08 smithi148 bash[59603]: 2e6a6f842c47f317e3b177da026caf78ade2f25a8f4925cf9aa84f2f7d13f7c0 2024-09-21T12:37:08.631 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:08 smithi148 systemd[1]: Started Ceph osd.7 for 6263deca-7813-11ef-baf6-efdc52797490. 2024-09-21T12:37:08.833 INFO:teuthology.orchestra.run.smithi078.stdout: 5 257370 9.10 MB/sec execute 586 sec latency 52.595 ms 2024-09-21T12:37:08.850 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:08 smithi078 ceph-mon[63866]: pgmap v296: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.0 MiB/s wr, 298 op/s; 254/1647 objects degraded (15.422%) 2024-09-21T12:37:08.851 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:08 smithi078 ceph-mon[63866]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T12:37:08.851 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:08 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:08.851 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:08 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:08.851 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:08 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:37:09.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:08 smithi148 ceph-mon[49387]: pgmap v296: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.0 MiB/s wr, 298 op/s; 254/1647 objects degraded (15.422%) 2024-09-21T12:37:09.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:08 smithi148 ceph-mon[49387]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T12:37:09.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:08 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:09.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:08 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:09.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:08 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:37:09.833 INFO:teuthology.orchestra.run.smithi078.stdout: 5 257568 9.10 MB/sec execute 587 sec latency 58.818 ms 2024-09-21T12:37:10.219 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:09 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:37:09.957+0000 7fb9a98a9640 -1 Falling back to public interface 2024-09-21T12:37:10.710 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:10 smithi078 ceph-mon[63866]: pgmap v297: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 344 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 4.0 MiB/s wr, 372 op/s; 254/1662 objects degraded (15.283%) 2024-09-21T12:37:10.833 INFO:teuthology.orchestra.run.smithi078.stdout: 5 257916 9.09 MB/sec execute 588 sec latency 55.547 ms 2024-09-21T12:37:11.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:10 smithi148 ceph-mon[49387]: pgmap v297: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 344 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 4.0 MiB/s wr, 372 op/s; 254/1662 objects degraded (15.283%) 2024-09-21T12:37:11.833 INFO:teuthology.orchestra.run.smithi078.stdout: 5 258222 9.09 MB/sec execute 589 sec latency 70.272 ms 2024-09-21T12:37:12.788 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:12 smithi078 ceph-mon[63866]: pgmap v298: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 337 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.9 MiB/s wr, 355 op/s; 252/1626 objects degraded (15.498%) 2024-09-21T12:37:12.789 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:12 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 252/1626 objects degraded (15.498%), 35 pgs degraded (PG_DEGRADED) 2024-09-21T12:37:12.789 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:12 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:12.789 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:12 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:12.833 INFO:teuthology.orchestra.run.smithi078.stdout: 5 258446 9.08 MB/sec execute 590 sec latency 49.763 ms 2024-09-21T12:37:13.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:12 smithi148 ceph-mon[49387]: pgmap v298: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 337 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.9 MiB/s wr, 355 op/s; 252/1626 objects degraded (15.498%) 2024-09-21T12:37:13.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:12 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 252/1626 objects degraded (15.498%), 35 pgs degraded (PG_DEGRADED) 2024-09-21T12:37:13.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:12 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:13.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:12 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:13.833 INFO:teuthology.orchestra.run.smithi078.stdout: 5 258709 9.08 MB/sec execute 591 sec latency 40.086 ms 2024-09-21T12:37:14.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:13 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:14.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:13 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:14.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:13 smithi148 ceph-mon[49387]: pgmap v299: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.2 MiB/s wr, 446 op/s; 257/1632 objects degraded (15.748%) 2024-09-21T12:37:14.070 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:13 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:14.070 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:13 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:14.070 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:13 smithi078 ceph-mon[63866]: pgmap v299: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.2 MiB/s wr, 446 op/s; 257/1632 objects degraded (15.748%) 2024-09-21T12:37:14.833 INFO:teuthology.orchestra.run.smithi078.stdout: 5 258995 9.08 MB/sec execute 592 sec latency 62.013 ms 2024-09-21T12:37:15.833 INFO:teuthology.orchestra.run.smithi078.stdout: 5 259337 9.08 MB/sec execute 593 sec latency 61.007 ms 2024-09-21T12:37:15.913 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:15 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:15.913 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:15 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:37:16.210 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:15 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:16.210 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:15 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:37:16.833 INFO:teuthology.orchestra.run.smithi078.stdout: 5 259565 9.08 MB/sec execute 594 sec latency 48.663 ms 2024-09-21T12:37:16.936 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:16 smithi078 ceph-mon[63866]: pgmap v300: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.0 MiB/s wr, 305 op/s; 257/1632 objects degraded (15.748%) 2024-09-21T12:37:16.936 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:16 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:16.937 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:16 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:16.937 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:16 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:37:16.937 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:16 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:37:16.937 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:16 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:16.937 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:16 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:17.191 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:16 smithi148 ceph-mon[49387]: pgmap v300: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.0 MiB/s wr, 305 op/s; 257/1632 objects degraded (15.748%) 2024-09-21T12:37:17.191 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:16 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:17.191 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:16 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:17.192 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:16 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:37:17.192 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:16 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:37:17.192 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:16 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:17.192 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:16 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:17.833 INFO:teuthology.orchestra.run.smithi078.stdout: 5 259788 9.08 MB/sec execute 595 sec latency 48.076 ms 2024-09-21T12:37:18.017 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:17 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 257/1632 objects degraded (15.748%), 35 pgs degraded (PG_DEGRADED) 2024-09-21T12:37:18.017 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:17 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:37:18.017 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:17 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:37:18.017 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:17 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:37:18.018 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:17 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:37:18.018 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:17 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:37:18.018 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:17 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:18.018 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:17 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-21T12:37:18.018 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:17 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-21T12:37:18.018 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:17 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-21T12:37:18.018 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:17 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-21T12:37:18.018 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:17 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-21T12:37:18.018 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:17 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-21T12:37:18.018 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:17 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-21T12:37:18.018 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:17 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-21T12:37:18.018 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:17 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-21T12:37:18.018 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:17 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-21T12:37:18.019 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:17 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-21T12:37:18.019 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:17 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-21T12:37:18.019 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:17 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-21T12:37:18.019 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:17 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-21T12:37:18.019 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:17 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-21T12:37:18.019 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:17 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-21T12:37:18.019 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:17 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-21T12:37:18.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:17 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 257/1632 objects degraded (15.748%), 35 pgs degraded (PG_DEGRADED) 2024-09-21T12:37:18.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:17 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:37:18.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:17 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:37:18.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:17 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:37:18.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:17 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:37:18.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:17 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:37:18.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:17 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:18.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:17 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-21T12:37:18.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:17 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-21T12:37:18.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:17 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-21T12:37:18.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:17 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-21T12:37:18.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:17 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-21T12:37:18.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:17 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-21T12:37:18.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:17 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-21T12:37:18.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:17 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-21T12:37:18.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:17 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-21T12:37:18.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:17 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-21T12:37:18.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:17 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-21T12:37:18.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:17 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-21T12:37:18.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:17 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-21T12:37:18.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:17 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-21T12:37:18.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:17 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-21T12:37:18.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:17 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-21T12:37:18.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:17 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-21T12:37:18.833 INFO:teuthology.orchestra.run.smithi078.stdout: 5 260097 9.08 MB/sec execute 596 sec latency 42.759 ms 2024-09-21T12:37:19.037 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:18 smithi078 ceph-mon[63866]: Upgrade: Setting container_image for all osd 2024-09-21T12:37:19.038 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:18 smithi078 ceph-mon[63866]: Upgrade: Setting require_osd_release to 19 squid 2024-09-21T12:37:19.038 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:18 smithi078 ceph-mon[63866]: pgmap v301: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.2 MiB/s wr, 358 op/s; 264/1644 objects degraded (16.058%) 2024-09-21T12:37:19.038 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:18 smithi078 ceph-mon[63866]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-21T12:37:19.038 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:18 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-21T12:37:19.038 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:18 smithi078 ceph-mon[63866]: osdmap e152: 8 total, 7 up, 8 in 2024-09-21T12:37:19.038 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:18 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi078.ytpzwh"]} : dispatch 2024-09-21T12:37:19.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:18 smithi148 ceph-mon[49387]: Upgrade: Setting container_image for all osd 2024-09-21T12:37:19.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:18 smithi148 ceph-mon[49387]: Upgrade: Setting require_osd_release to 19 squid 2024-09-21T12:37:19.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:18 smithi148 ceph-mon[49387]: pgmap v301: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.2 MiB/s wr, 358 op/s; 264/1644 objects degraded (16.058%) 2024-09-21T12:37:19.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:18 smithi148 ceph-mon[49387]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-21T12:37:19.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:18 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-21T12:37:19.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:18 smithi148 ceph-mon[49387]: osdmap e152: 8 total, 7 up, 8 in 2024-09-21T12:37:19.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:18 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi078.ytpzwh"]} : dispatch 2024-09-21T12:37:19.833 INFO:teuthology.orchestra.run.smithi078.stdout: 5 260417 9.08 MB/sec execute 597 sec latency 42.612 ms 2024-09-21T12:37:20.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:19 smithi148 ceph-mon[49387]: Upgrade: It appears safe to stop mds.foofs.smithi078.ytpzwh 2024-09-21T12:37:20.051 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:19 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:37:19.789+0000 7fb9a98a9640 -1 osd.7 0 read_superblock omap replica is missing. 2024-09-21T12:37:20.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:19 smithi078 ceph-mon[63866]: Upgrade: It appears safe to stop mds.foofs.smithi078.ytpzwh 2024-09-21T12:37:20.833 INFO:teuthology.orchestra.run.smithi078.stdout: 5 260694 9.07 MB/sec execute 598 sec latency 58.596 ms 2024-09-21T12:37:21.083 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:20 smithi078 ceph-mon[63866]: pgmap v303: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.7 MiB/s wr, 386 op/s; 263/1641 objects degraded (16.027%) 2024-09-21T12:37:21.084 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:20 smithi078 ceph-mon[63866]: Upgrade: Updating mds.foofs.smithi078.ytpzwh 2024-09-21T12:37:21.084 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:20 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:21.084 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:20 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.ytpzwh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T12:37:21.084 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:20 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:37:21.084 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:20 smithi078 ceph-mon[63866]: Deploying daemon mds.foofs.smithi078.ytpzwh on smithi078 2024-09-21T12:37:21.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:20 smithi148 ceph-mon[49387]: pgmap v303: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.7 MiB/s wr, 386 op/s; 263/1641 objects degraded (16.027%) 2024-09-21T12:37:21.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:20 smithi148 ceph-mon[49387]: Upgrade: Updating mds.foofs.smithi078.ytpzwh 2024-09-21T12:37:21.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:20 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:21.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:20 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.ytpzwh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T12:37:21.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:20 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:37:21.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:20 smithi148 ceph-mon[49387]: Deploying daemon mds.foofs.smithi078.ytpzwh on smithi078 2024-09-21T12:37:21.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:21 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:37:21.355+0000 7fb9a98a9640 -1 osd.7 149 log_to_monitors true 2024-09-21T12:37:21.833 INFO:teuthology.orchestra.run.smithi078.stdout: 5 260940 9.06 MB/sec execute 599 sec latency 64.412 ms 2024-09-21T12:37:22.126 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:21 smithi078 ceph-mon[63866]: from='osd.7 [v2:172.21.15.148:6824/3472878116,v1:172.21.15.148:6825/3472878116]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-21T12:37:22.127 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:21 smithi078 ceph-mon[63866]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-21T12:37:22.127 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:21 smithi078 ceph-mon[63866]: pgmap v304: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.9 MiB/s wr, 389 op/s; 264/1647 objects degraded (16.029%) 2024-09-21T12:37:22.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:21 smithi148 ceph-mon[49387]: from='osd.7 [v2:172.21.15.148:6824/3472878116,v1:172.21.15.148:6825/3472878116]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-21T12:37:22.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:21 smithi148 ceph-mon[49387]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-21T12:37:22.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:21 smithi148 ceph-mon[49387]: pgmap v304: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.9 MiB/s wr, 389 op/s; 264/1647 objects degraded (16.029%) 2024-09-21T12:37:22.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:22 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:37:22.388+0000 7fb99d64b640 -1 osd.7 149 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T12:37:22.833 INFO:teuthology.orchestra.run.smithi078.stdout: 5 cleanup 600 sec 2024-09-21T12:37:23.086 INFO:teuthology.orchestra.run.smithi078.stdout: 0 cleanup 600 sec 2024-09-21T12:37:23.086 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T12:37:23.086 INFO:teuthology.orchestra.run.smithi078.stdout: Operation Count AvgLat MaxLat 2024-09-21T12:37:23.086 INFO:teuthology.orchestra.run.smithi078.stdout: ---------------------------------------- 2024-09-21T12:37:23.086 INFO:teuthology.orchestra.run.smithi078.stdout: NTCreateX 174353 4.741 1060.138 2024-09-21T12:37:23.087 INFO:teuthology.orchestra.run.smithi078.stdout: Close 127902 0.128 10.815 2024-09-21T12:37:23.087 INFO:teuthology.orchestra.run.smithi078.stdout: Rename 7389 9.170 1051.112 2024-09-21T12:37:23.087 INFO:teuthology.orchestra.run.smithi078.stdout: Unlink 35343 7.136 1040.294 2024-09-21T12:37:23.087 INFO:teuthology.orchestra.run.smithi078.stdout: Qpathinfo 157998 1.252 1110.557 2024-09-21T12:37:23.087 INFO:teuthology.orchestra.run.smithi078.stdout: Qfileinfo 27513 0.007 3.725 2024-09-21T12:37:23.087 INFO:teuthology.orchestra.run.smithi078.stdout: Qfsinfo 29043 0.558 79.877 2024-09-21T12:37:23.087 INFO:teuthology.orchestra.run.smithi078.stdout: Sfileinfo 14201 13.848 1036.274 2024-09-21T12:37:23.087 INFO:teuthology.orchestra.run.smithi078.stdout: Find 61054 0.889 23.921 2024-09-21T12:37:23.087 INFO:teuthology.orchestra.run.smithi078.stdout: WriteX 86075 15.867 2544.436 2024-09-21T12:37:23.087 INFO:teuthology.orchestra.run.smithi078.stdout: ReadX 273585 0.010 4.415 2024-09-21T12:37:23.087 INFO:teuthology.orchestra.run.smithi078.stdout: LockX 570 0.344 2.734 2024-09-21T12:37:23.087 INFO:teuthology.orchestra.run.smithi078.stdout: UnlockX 570 0.307 1.709 2024-09-21T12:37:23.087 INFO:teuthology.orchestra.run.smithi078.stdout: Flush 12226 0.002 0.050 2024-09-21T12:37:23.087 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T12:37:23.088 INFO:teuthology.orchestra.run.smithi078.stdout:Throughput 9.06431 MB/sec 5 clients 5 procs max_latency=2544.444 ms 2024-09-21T12:37:23.090 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-09-21T12:37:23.128 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-09-21T12:37:23.188 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:22 smithi078 ceph-mon[63866]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T12:37:23.188 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:22 smithi078 ceph-mon[63866]: from='osd.7 [v2:172.21.15.148:6824/3472878116,v1:172.21.15.148:6825/3472878116]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi148", "root=default"]} : dispatch 2024-09-21T12:37:23.188 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:22 smithi078 ceph-mon[63866]: osdmap e153: 8 total, 7 up, 8 in 2024-09-21T12:37:23.188 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:22 smithi078 ceph-mon[63866]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi148", "root=default"]} : dispatch 2024-09-21T12:37:23.188 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:22 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 264/1647 objects degraded (16.029%), 35 pgs degraded (PG_DEGRADED) 2024-09-21T12:37:23.188 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:22 smithi078 ceph-mon[63866]: from='osd.7 ' entity='osd.7' 2024-09-21T12:37:23.270 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-09-21T12:37:23.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:22 smithi148 ceph-mon[49387]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T12:37:23.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:22 smithi148 ceph-mon[49387]: from='osd.7 [v2:172.21.15.148:6824/3472878116,v1:172.21.15.148:6825/3472878116]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi148", "root=default"]} : dispatch 2024-09-21T12:37:23.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:22 smithi148 ceph-mon[49387]: osdmap e153: 8 total, 7 up, 8 in 2024-09-21T12:37:23.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:22 smithi148 ceph-mon[49387]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi148", "root=default"]} : dispatch 2024-09-21T12:37:23.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:22 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 264/1647 objects degraded (16.029%), 35 pgs degraded (PG_DEGRADED) 2024-09-21T12:37:23.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:22 smithi148 ceph-mon[49387]: from='osd.7 ' entity='osd.7' 2024-09-21T12:37:23.305 INFO:teuthology.orchestra.run.smithi078.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-21T12:37:23.305 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T12:37:23.305 INFO:teuthology.orchestra.run.smithi078.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-09-21T12:37:23.306 INFO:teuthology.orchestra.run.smithi078.stdout:3 of 5 processes prepared for launch 0 sec 2024-09-21T12:37:23.406 INFO:teuthology.orchestra.run.smithi078.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-21T12:37:23.406 INFO:teuthology.orchestra.run.smithi078.stdout:releasing clients 2024-09-21T12:37:24.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:23 smithi148 ceph-mon[49387]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T12:37:24.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:23 smithi148 ceph-mon[49387]: osd.7 [v2:172.21.15.148:6824/3472878116,v1:172.21.15.148:6825/3472878116] boot 2024-09-21T12:37:24.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:23 smithi148 ceph-mon[49387]: osdmap e154: 8 total, 8 up, 8 in 2024-09-21T12:37:24.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:23 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-21T12:37:24.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:23 smithi148 ceph-mon[49387]: pgmap v307: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 346 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.7 MiB/s wr, 502 op/s; 271/1686 objects degraded (16.074%) 2024-09-21T12:37:24.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:23 smithi078 ceph-mon[63866]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T12:37:24.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:23 smithi078 ceph-mon[63866]: osd.7 [v2:172.21.15.148:6824/3472878116,v1:172.21.15.148:6825/3472878116] boot 2024-09-21T12:37:24.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:23 smithi078 ceph-mon[63866]: osdmap e154: 8 total, 8 up, 8 in 2024-09-21T12:37:24.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:23 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-21T12:37:24.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:23 smithi078 ceph-mon[63866]: pgmap v307: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 346 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.7 MiB/s wr, 502 op/s; 271/1686 objects degraded (16.074%) 2024-09-21T12:37:24.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:23 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi078[63862]: 2024-09-21T12:37:23.911+0000 7f79f2431640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T12:37:24.406 INFO:teuthology.orchestra.run.smithi078.stdout: 0 8 0.00 MB/sec warmup 1 sec latency 999.935 ms 2024-09-21T12:37:25.300 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:37:25 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:37:25.055+0000 7f83faa6d640 -1 bdev(0x55d3577e2c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 4 2024-09-21T12:37:25.300 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:37:25 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:37:25.058+0000 7f83fb26e640 -1 bdev(0x55d3577e2c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 5 2024-09-21T12:37:25.300 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:37:25 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:37:25.058+0000 7f8400278640 -1 bdev(0x55d3577e2c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 5 2024-09-21T12:37:25.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:24 smithi148 ceph-mon[49387]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-21T12:37:25.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:24 smithi148 ceph-mon[49387]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T12:37:25.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:24 smithi148 ceph-mon[49387]: osdmap e155: 8 total, 8 up, 8 in 2024-09-21T12:37:25.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:24 smithi148 ceph-mon[49387]: Standby daemon mds.foofs.smithi148.uukezr assigned to filesystem foofs as rank 0 2024-09-21T12:37:25.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:24 smithi148 ceph-mon[49387]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-21T12:37:25.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:24 smithi148 ceph-mon[49387]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T12:37:25.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:24 smithi148 ceph-mon[49387]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-21T12:37:25.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:24 smithi148 ceph-mon[49387]: fsmap foofs:1/1 {0=foofs.smithi148.uukezr=up:replay} 2024-09-21T12:37:25.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:24 smithi078 ceph-mon[63866]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-21T12:37:25.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:24 smithi078 ceph-mon[63866]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T12:37:25.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:24 smithi078 ceph-mon[63866]: osdmap e155: 8 total, 8 up, 8 in 2024-09-21T12:37:25.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:24 smithi078 ceph-mon[63866]: Standby daemon mds.foofs.smithi148.uukezr assigned to filesystem foofs as rank 0 2024-09-21T12:37:25.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:24 smithi078 ceph-mon[63866]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-21T12:37:25.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:24 smithi078 ceph-mon[63866]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T12:37:25.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:24 smithi078 ceph-mon[63866]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-21T12:37:25.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:24 smithi078 ceph-mon[63866]: fsmap foofs:1/1 {0=foofs.smithi148.uukezr=up:replay} 2024-09-21T12:37:25.406 INFO:teuthology.orchestra.run.smithi078.stdout: 0 8 0.00 MB/sec warmup 2 sec latency 1999.965 ms 2024-09-21T12:37:26.259 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:25 smithi078 ceph-mon[63866]: osdmap e156: 8 total, 8 up, 8 in 2024-09-21T12:37:26.259 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:25 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mds metadata", "who": "foofs.smithi078.ytpzwh"} : dispatch 2024-09-21T12:37:26.259 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:25 smithi078 ceph-mon[63866]: pgmap v310: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 346 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.3 MiB/s wr, 374 op/s; 271/1686 objects degraded (16.074%) 2024-09-21T12:37:26.260 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-21T12:37:26.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:25 smithi148 ceph-mon[49387]: osdmap e156: 8 total, 8 up, 8 in 2024-09-21T12:37:26.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:25 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mds metadata", "who": "foofs.smithi078.ytpzwh"} : dispatch 2024-09-21T12:37:26.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:25 smithi148 ceph-mon[49387]: pgmap v310: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 346 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.3 MiB/s wr, 374 op/s; 271/1686 objects degraded (16.074%) 2024-09-21T12:37:26.406 INFO:teuthology.orchestra.run.smithi078.stdout: 0 8 0.00 MB/sec warmup 3 sec latency 2999.995 ms 2024-09-21T12:37:26.971 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T12:37:26.971 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (9m) 2m ago 18m 20.0M - 0.27.0 11f11916f8cd e6e67ab07dec 2024-09-21T12:37:26.971 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (6m) 2m ago 18m 6895k - 19.3.0-5104-g44178494 b415816db8b4 46a10ea0b111 2024-09-21T12:37:26.971 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi148 smithi148 running (6m) 14s ago 18m 6895k - 19.3.0-5104-g44178494 b415816db8b4 23deb0615621 2024-09-21T12:37:26.971 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (9m) 2m ago 18m 53.3M - dad864ee21e9 2a140c36091b 2024-09-21T12:37:26.971 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.ytpzwh smithi078 running (12m) 2m ago 12m 38.4M - 17.2.0 e1d6a67b021e 5e9b109502a2 2024-09-21T12:37:26.971 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi148.uukezr smithi148 running (12m) 14s ago 12m 14.8M - 17.2.0 e1d6a67b021e b7e8aa0f3a6c 2024-09-21T12:37:26.971 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.ntwyan smithi078 *:8443,9283,8765 running (8m) 2m ago 19m 519M - 19.3.0-5104-g44178494 b415816db8b4 a61baedebf43 2024-09-21T12:37:26.971 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi148.oyiqmx smithi148 *:8443,9283,8765 running (7m) 14s ago 17m 458M - 19.3.0-5104-g44178494 b415816db8b4 0255535c6fdb 2024-09-21T12:37:26.971 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (7m) 2m ago 20m 55.9M 2048M 19.3.0-5104-g44178494 b415816db8b4 22b3097efbd8 2024-09-21T12:37:26.971 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi148 smithi148 running (7m) 14s ago 17m 46.6M 2048M 19.3.0-5104-g44178494 b415816db8b4 df8b08509136 2024-09-21T12:37:26.972 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.ydbhru smithi078 *:2049 running (12m) 2m ago 12m 279M - 4.0 e1d6a67b021e 7e5b0274ac38 2024-09-21T12:37:26.972 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi148.einuzt smithi148 *:2049 running (12m) 14s ago 12m 87.3M - 4.0 e1d6a67b021e 76a8b0eabede 2024-09-21T12:37:26.972 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (9m) 2m ago 18m 11.0M - 1.7.0 72c9c2088986 301adb4e9e22 2024-09-21T12:37:26.972 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi148 smithi148 *:9100 running (8m) 14s ago 17m 12.3M - 1.7.0 72c9c2088986 0cceff4bf342 2024-09-21T12:37:26.972 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (5m) 2m ago 17m 345M 1936M 19.3.0-5104-g44178494 b415816db8b4 7374add97534 2024-09-21T12:37:26.972 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (4m) 2m ago 16m 276M 1936M 19.3.0-5104-g44178494 b415816db8b4 14655a8da4e5 2024-09-21T12:37:26.972 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (3m) 2m ago 16m 219M 1936M 19.3.0-5104-g44178494 b415816db8b4 07ceda32f383 2024-09-21T12:37:26.972 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (3m) 2m ago 15m 11.4M 1936M 19.3.0-5104-g44178494 b415816db8b4 ead87e4306ff 2024-09-21T12:37:26.972 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi148 running (2m) 14s ago 15m 331M 2704M 19.3.0-5104-g44178494 b415816db8b4 10ec7d6b476e 2024-09-21T12:37:26.972 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi148 running (97s) 14s ago 14m 298M 2704M 19.3.0-5104-g44178494 b415816db8b4 06ac02700ea1 2024-09-21T12:37:26.972 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi148 running (57s) 14s ago 14m 246M 2704M 19.3.0-5104-g44178494 b415816db8b4 17613684a653 2024-09-21T12:37:26.972 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi148 running (19s) 14s ago 14m 14.4M 2704M 19.3.0-5104-g44178494 b415816db8b4 2e6a6f842c47 2024-09-21T12:37:26.972 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (7m) 2m ago 17m 55.8M - 2.51.0 1d3b7f56885b face2d4cdd1b 2024-09-21T12:37:27.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:26 smithi148 ceph-mon[49387]: osdmap e157: 8 total, 8 up, 8 in 2024-09-21T12:37:27.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:26 smithi148 ceph-mon[49387]: from='client.34450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:37:27.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:26 smithi148 ceph-mon[49387]: from='client.34458 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:37:27.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:26 smithi078 ceph-mon[63866]: osdmap e157: 8 total, 8 up, 8 in 2024-09-21T12:37:27.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:26 smithi078 ceph-mon[63866]: from='client.34450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:37:27.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:26 smithi078 ceph-mon[63866]: from='client.34458 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:37:27.403 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:37:27.403 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-21T12:37:27.404 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:37:27.404 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:37:27.404 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-21T12:37:27.404 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:37:27.404 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:37:27.404 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-21T12:37:27.404 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 8 2024-09-21T12:37:27.404 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:37:27.404 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-21T12:37:27.404 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-21T12:37:27.404 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:37:27.404 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-21T12:37:27.404 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-21T12:37:27.404 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 12 2024-09-21T12:37:27.404 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T12:37:27.405 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:37:27.406 INFO:teuthology.orchestra.run.smithi078.stdout: 0 8 0.00 MB/sec warmup 4 sec latency 4000.025 ms 2024-09-21T12:37:27.764 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:37:27.764 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5", 2024-09-21T12:37:27.764 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-21T12:37:27.764 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T12:37:27.764 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-21T12:37:27.765 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-21T12:37:27.765 INFO:teuthology.orchestra.run.smithi078.stdout: "osd", 2024-09-21T12:37:27.765 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-09-21T12:37:27.765 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr" 2024-09-21T12:37:27.765 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-21T12:37:27.765 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "14/23 daemons upgraded", 2024-09-21T12:37:27.765 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading mds daemons", 2024-09-21T12:37:27.765 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-21T12:37:27.765 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:37:28.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:27 smithi078 ceph-mon[63866]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-21T12:37:28.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:27 smithi078 ceph-mon[63866]: from='client.34462 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:37:28.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:27 smithi078 ceph-mon[63866]: osdmap e158: 8 total, 8 up, 8 in 2024-09-21T12:37:28.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:27 smithi078 ceph-mon[63866]: mds.? [v2:172.21.15.78:6834/3483195741,v1:172.21.15.78:6835/3483195741] up:boot 2024-09-21T12:37:28.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:27 smithi078 ceph-mon[63866]: fsmap foofs:1/1 {0=foofs.smithi148.uukezr=up:replay} 1 up:standby 2024-09-21T12:37:28.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:27 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mds metadata", "who": "foofs.smithi078.ytpzwh"} : dispatch 2024-09-21T12:37:28.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:27 smithi078 ceph-mon[63866]: Health check update: Degraded data redundancy: 271/1686 objects degraded (16.074%), 35 pgs degraded (PG_DEGRADED) 2024-09-21T12:37:28.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:27 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/3617711246' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:37:28.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:27 smithi078 ceph-mon[63866]: pgmap v313: 97 pgs: 2 peering, 1 active+undersized+remapped, 2 active+undersized, 4 active+undersized+degraded, 88 active+clean; 344 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 35 MiB/s rd, 1.5 MiB/s wr, 213 op/s; 33/1554 objects degraded (2.124%); 3.3 MiB/s, 66 objects/s recovering 2024-09-21T12:37:28.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:27 smithi078 ceph-mon[63866]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:37:28.205 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_WARN 1 filesystem is degraded; Degraded data redundancy: 33/1554 objects degraded (2.124%), 4 pgs degraded 2024-09-21T12:37:28.205 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] FS_DEGRADED: 1 filesystem is degraded 2024-09-21T12:37:28.205 INFO:teuthology.orchestra.run.smithi078.stdout: fs foofs is degraded 2024-09-21T12:37:28.205 INFO:teuthology.orchestra.run.smithi078.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 33/1554 objects degraded (2.124%), 4 pgs degraded 2024-09-21T12:37:28.205 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.3 is active+undersized+degraded, acting [5,2] 2024-09-21T12:37:28.205 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.e is active+undersized+degraded, acting [2,3] 2024-09-21T12:37:28.205 INFO:teuthology.orchestra.run.smithi078.stdout: pg 2.12 is active+undersized+degraded, acting [5,3] 2024-09-21T12:37:28.205 INFO:teuthology.orchestra.run.smithi078.stdout: pg 3.16 is active+undersized+degraded, acting [5,1] 2024-09-21T12:37:28.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:27 smithi148 ceph-mon[49387]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-21T12:37:28.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:27 smithi148 ceph-mon[49387]: from='client.34462 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:37:28.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:27 smithi148 ceph-mon[49387]: osdmap e158: 8 total, 8 up, 8 in 2024-09-21T12:37:28.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:27 smithi148 ceph-mon[49387]: mds.? [v2:172.21.15.78:6834/3483195741,v1:172.21.15.78:6835/3483195741] up:boot 2024-09-21T12:37:28.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:27 smithi148 ceph-mon[49387]: fsmap foofs:1/1 {0=foofs.smithi148.uukezr=up:replay} 1 up:standby 2024-09-21T12:37:28.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:27 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mds metadata", "who": "foofs.smithi078.ytpzwh"} : dispatch 2024-09-21T12:37:28.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:27 smithi148 ceph-mon[49387]: Health check update: Degraded data redundancy: 271/1686 objects degraded (16.074%), 35 pgs degraded (PG_DEGRADED) 2024-09-21T12:37:28.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:27 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/3617711246' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:37:28.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:27 smithi148 ceph-mon[49387]: pgmap v313: 97 pgs: 2 peering, 1 active+undersized+remapped, 2 active+undersized, 4 active+undersized+degraded, 88 active+clean; 344 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 35 MiB/s rd, 1.5 MiB/s wr, 213 op/s; 33/1554 objects degraded (2.124%); 3.3 MiB/s, 66 objects/s recovering 2024-09-21T12:37:28.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:27 smithi148 ceph-mon[49387]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:37:28.407 INFO:teuthology.orchestra.run.smithi078.stdout: 0 8 0.00 MB/sec warmup 5 sec latency 5000.081 ms 2024-09-21T12:37:29.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:29 smithi148 ceph-mon[49387]: reconnect by client.24327 172.21.15.148:0/3482514271 after 0.000999976 2024-09-21T12:37:29.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:29 smithi148 ceph-mon[49387]: mds.? [v2:172.21.15.148:6832/3237004392,v1:172.21.15.148:6833/3237004392] up:reconnect 2024-09-21T12:37:29.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:29 smithi148 ceph-mon[49387]: fsmap foofs:1/1 {0=foofs.smithi148.uukezr=up:reconnect} 1 up:standby 2024-09-21T12:37:29.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:29 smithi148 ceph-mon[49387]: reconnect by client.14556 172.21.15.78:0/1840377618 after 0.0239994 2024-09-21T12:37:29.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:29 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/2067424993' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:37:29.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:29 smithi078 ceph-mon[63866]: reconnect by client.24327 172.21.15.148:0/3482514271 after 0.000999976 2024-09-21T12:37:29.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:29 smithi078 ceph-mon[63866]: mds.? [v2:172.21.15.148:6832/3237004392,v1:172.21.15.148:6833/3237004392] up:reconnect 2024-09-21T12:37:29.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:29 smithi078 ceph-mon[63866]: fsmap foofs:1/1 {0=foofs.smithi148.uukezr=up:reconnect} 1 up:standby 2024-09-21T12:37:29.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:29 smithi078 ceph-mon[63866]: reconnect by client.14556 172.21.15.78:0/1840377618 after 0.0239994 2024-09-21T12:37:29.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:29 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/2067424993' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:37:29.407 INFO:teuthology.orchestra.run.smithi078.stdout: 0 8 0.00 MB/sec warmup 6 sec latency 6000.109 ms 2024-09-21T12:37:30.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:30 smithi148 ceph-mon[49387]: mds.? [v2:172.21.15.148:6832/3237004392,v1:172.21.15.148:6833/3237004392] up:rejoin 2024-09-21T12:37:30.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:30 smithi148 ceph-mon[49387]: fsmap foofs:1/1 {0=foofs.smithi148.uukezr=up:rejoin} 1 up:standby 2024-09-21T12:37:30.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:30 smithi148 ceph-mon[49387]: pgmap v314: 97 pgs: 2 peering, 1 active+undersized+remapped, 94 active+clean; 343 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 38 MiB/s rd, 1.1 MiB/s wr, 156 op/s; 3.0 MiB/s, 51 objects/s recovering 2024-09-21T12:37:30.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:30.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:37:30.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:30 smithi078 ceph-mon[63866]: mds.? [v2:172.21.15.148:6832/3237004392,v1:172.21.15.148:6833/3237004392] up:rejoin 2024-09-21T12:37:30.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:30 smithi078 ceph-mon[63866]: fsmap foofs:1/1 {0=foofs.smithi148.uukezr=up:rejoin} 1 up:standby 2024-09-21T12:37:30.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:30 smithi078 ceph-mon[63866]: pgmap v314: 97 pgs: 2 peering, 1 active+undersized+remapped, 94 active+clean; 343 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 38 MiB/s rd, 1.1 MiB/s wr, 156 op/s; 3.0 MiB/s, 51 objects/s recovering 2024-09-21T12:37:30.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:30.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:37:30.407 INFO:teuthology.orchestra.run.smithi078.stdout: 0 8 0.00 MB/sec warmup 7 sec latency 7000.160 ms 2024-09-21T12:37:31.284 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:31 smithi078 ceph-mon[63866]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 33/1554 objects degraded (2.124%), 4 pgs degraded) 2024-09-21T12:37:31.285 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:31 smithi078 ceph-mon[63866]: mds.? [v2:172.21.15.148:6832/3237004392,v1:172.21.15.148:6833/3237004392] up:clientreplay 2024-09-21T12:37:31.285 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:31 smithi078 ceph-mon[63866]: fsmap foofs:1/1 {0=foofs.smithi148.uukezr=up:clientreplay} 1 up:standby 2024-09-21T12:37:31.285 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:31 smithi078 ceph-mon[63866]: daemon mds.foofs.smithi148.uukezr is now active in filesystem foofs as rank 0 2024-09-21T12:37:31.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:31 smithi148 ceph-mon[49387]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 33/1554 objects degraded (2.124%), 4 pgs degraded) 2024-09-21T12:37:31.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:31 smithi148 ceph-mon[49387]: mds.? [v2:172.21.15.148:6832/3237004392,v1:172.21.15.148:6833/3237004392] up:clientreplay 2024-09-21T12:37:31.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:31 smithi148 ceph-mon[49387]: fsmap foofs:1/1 {0=foofs.smithi148.uukezr=up:clientreplay} 1 up:standby 2024-09-21T12:37:31.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:31 smithi148 ceph-mon[49387]: daemon mds.foofs.smithi148.uukezr is now active in filesystem foofs as rank 0 2024-09-21T12:37:32.081 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:32 smithi078 ceph-mon[63866]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-21T12:37:32.082 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:32 smithi078 ceph-mon[63866]: Cluster is now healthy 2024-09-21T12:37:32.082 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:32 smithi078 ceph-mon[63866]: mds.? [v2:172.21.15.148:6832/3237004392,v1:172.21.15.148:6833/3237004392] up:active 2024-09-21T12:37:32.082 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:32 smithi078 ceph-mon[63866]: fsmap foofs:1 {0=foofs.smithi148.uukezr=up:active} 1 up:standby 2024-09-21T12:37:32.082 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:32 smithi078 ceph-mon[63866]: pgmap v315: 97 pgs: 1 peering, 96 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 2.1 MiB/s wr, 153 op/s; 2.5 MiB/s, 44 objects/s recovering 2024-09-21T12:37:32.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:32 smithi148 ceph-mon[49387]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-21T12:37:32.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:32 smithi148 ceph-mon[49387]: Cluster is now healthy 2024-09-21T12:37:32.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:32 smithi148 ceph-mon[49387]: mds.? [v2:172.21.15.148:6832/3237004392,v1:172.21.15.148:6833/3237004392] up:active 2024-09-21T12:37:32.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:32 smithi148 ceph-mon[49387]: fsmap foofs:1 {0=foofs.smithi148.uukezr=up:active} 1 up:standby 2024-09-21T12:37:32.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:32 smithi148 ceph-mon[49387]: pgmap v315: 97 pgs: 1 peering, 96 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 2.1 MiB/s wr, 153 op/s; 2.5 MiB/s, 44 objects/s recovering 2024-09-21T12:37:32.407 INFO:teuthology.orchestra.run.smithi078.stdout: 5 172 21.69 MB/sec execute 1 sec latency 188.786 ms 2024-09-21T12:37:33.407 INFO:teuthology.orchestra.run.smithi078.stdout: 5 233 15.65 MB/sec execute 2 sec latency 123.008 ms 2024-09-21T12:37:34.407 INFO:teuthology.orchestra.run.smithi078.stdout: 5 313 14.60 MB/sec execute 3 sec latency 125.874 ms 2024-09-21T12:37:34.827 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:34 smithi078 ceph-mon[63866]: pgmap v316: 97 pgs: 97 active+clean; 262 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 3.6 MiB/s wr, 192 op/s; 2.1 MiB/s, 36 objects/s recovering 2024-09-21T12:37:35.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:34 smithi148 ceph-mon[49387]: pgmap v316: 97 pgs: 97 active+clean; 262 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 3.6 MiB/s wr, 192 op/s; 2.1 MiB/s, 36 objects/s recovering 2024-09-21T12:37:35.407 INFO:teuthology.orchestra.run.smithi078.stdout: 5 383 13.68 MB/sec execute 4 sec latency 118.143 ms 2024-09-21T12:37:36.407 INFO:teuthology.orchestra.run.smithi078.stdout: 5 cleanup 5 sec 2024-09-21T12:37:36.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:36 smithi078 ceph-mon[63866]: pgmap v317: 97 pgs: 97 active+clean; 262 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.5 MiB/s wr, 87 op/s; 1.3 MiB/s, 27 objects/s recovering 2024-09-21T12:37:36.727 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:36 smithi078 ceph-mon[63866]: daemon mds.foofs.smithi078.ytpzwh restarted 2024-09-21T12:37:36.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:36 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:36.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:36 smithi078 ceph-mon[63866]: osdmap e159: 8 total, 8 up, 8 in 2024-09-21T12:37:36.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:36 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:36.728 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:36 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:37:37.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:36 smithi148 ceph-mon[49387]: pgmap v317: 97 pgs: 97 active+clean; 262 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.5 MiB/s wr, 87 op/s; 1.3 MiB/s, 27 objects/s recovering 2024-09-21T12:37:37.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:36 smithi148 ceph-mon[49387]: daemon mds.foofs.smithi078.ytpzwh restarted 2024-09-21T12:37:37.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:36 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:37.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:36 smithi148 ceph-mon[49387]: osdmap e159: 8 total, 8 up, 8 in 2024-09-21T12:37:37.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:36 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:37.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:36 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:37:37.217 INFO:teuthology.orchestra.run.smithi078.stdout: 0 cleanup 6 sec 2024-09-21T12:37:37.217 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T12:37:37.217 INFO:teuthology.orchestra.run.smithi078.stdout: Operation Count AvgLat MaxLat 2024-09-21T12:37:37.217 INFO:teuthology.orchestra.run.smithi078.stdout: ---------------------------------------- 2024-09-21T12:37:37.218 INFO:teuthology.orchestra.run.smithi078.stdout: NTCreateX 265 57.872 125.870 2024-09-21T12:37:37.218 INFO:teuthology.orchestra.run.smithi078.stdout: Close 263 22.478 188.777 2024-09-21T12:37:37.218 INFO:teuthology.orchestra.run.smithi078.stdout: Qfileinfo 226 18.977 60.044 2024-09-21T12:37:37.218 INFO:teuthology.orchestra.run.smithi078.stdout: WriteX 1274 0.028 0.079 2024-09-21T12:37:37.218 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T12:37:37.218 INFO:teuthology.orchestra.run.smithi078.stdout:Throughput 13.6793 MB/sec 5 clients 5 procs max_latency=188.786 ms 2024-09-21T12:37:37.219 DEBUG:teuthology.parallel:result is None 2024-09-21T12:37:37.904 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:37 smithi078 ceph-mon[63866]: mds.? [v2:172.21.15.78:6834/4075310113,v1:172.21.15.78:6835/4075310113] up:boot 2024-09-21T12:37:37.904 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:37 smithi078 ceph-mon[63866]: fsmap foofs:1 {0=foofs.smithi148.uukezr=up:active} 1 up:standby 2024-09-21T12:37:37.904 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:37 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mds metadata", "who": "foofs.smithi078.ytpzwh"} : dispatch 2024-09-21T12:37:37.952 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:37 smithi148 ceph-mon[49387]: mds.? [v2:172.21.15.78:6834/4075310113,v1:172.21.15.78:6835/4075310113] up:boot 2024-09-21T12:37:37.952 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:37 smithi148 ceph-mon[49387]: fsmap foofs:1 {0=foofs.smithi148.uukezr=up:active} 1 up:standby 2024-09-21T12:37:37.952 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:37 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mds metadata", "who": "foofs.smithi078.ytpzwh"} : dispatch 2024-09-21T12:37:38.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:38 smithi078 ceph-mon[63866]: pgmap v319: 97 pgs: 97 active+clean; 311 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 7.6 MiB/s wr, 190 op/s; 363 KiB/s, 2 objects/s recovering 2024-09-21T12:37:38.969 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:38 smithi148 ceph-mon[49387]: pgmap v319: 97 pgs: 97 active+clean; 311 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.8 MiB/s rd, 7.6 MiB/s wr, 190 op/s; 363 KiB/s, 2 objects/s recovering 2024-09-21T12:37:40.876 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:40 smithi078 ceph-mon[63866]: pgmap v320: 97 pgs: 97 active+clean; 317 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 8.2 MiB/s wr, 193 op/s 2024-09-21T12:37:40.876 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:40 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:41.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:40 smithi148 ceph-mon[49387]: pgmap v320: 97 pgs: 97 active+clean; 317 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 8.2 MiB/s wr, 193 op/s 2024-09-21T12:37:41.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:40 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:42.405 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:42 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:42.405 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:42 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:42.405 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:42 smithi078 ceph-mon[63866]: pgmap v321: 97 pgs: 97 active+clean; 291 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 7.5 MiB/s wr, 188 op/s 2024-09-21T12:37:42.405 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:42 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:42.405 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:42 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:42.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:42 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:42.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:42 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:42.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:42 smithi148 ceph-mon[49387]: pgmap v321: 97 pgs: 97 active+clean; 291 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 7.5 MiB/s wr, 188 op/s 2024-09-21T12:37:42.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:42 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:42.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:42 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:44.909 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:44 smithi078 ceph-mon[63866]: pgmap v322: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 6.1 MiB/s wr, 155 op/s 2024-09-21T12:37:45.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:44 smithi148 ceph-mon[49387]: pgmap v322: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 6.1 MiB/s wr, 155 op/s 2024-09-21T12:37:46.129 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:46.129 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:37:46.129 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:46.129 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:46.129 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:37:46.129 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:37:46.129 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:46.129 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:46.129 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:37:46.129 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:37:46.129 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:37:46.129 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:37:46.130 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:37:46.130 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi148.uukezr"]} : dispatch 2024-09-21T12:37:46.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:46.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:37:46.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:46.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:46.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:37:46.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:37:46.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:46.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:46.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:37:46.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:37:46.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:37:46.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:37:46.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:37:46.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi148.uukezr"]} : dispatch 2024-09-21T12:37:47.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:46 smithi148 ceph-mon[49387]: Detected new or changed devices on smithi078 2024-09-21T12:37:47.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:46 smithi148 ceph-mon[49387]: pgmap v323: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 6.1 MiB/s wr, 155 op/s 2024-09-21T12:37:47.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:46 smithi148 ceph-mon[49387]: Upgrade: It appears safe to stop mds.foofs.smithi148.uukezr 2024-09-21T12:37:47.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:46 smithi078 ceph-mon[63866]: Detected new or changed devices on smithi078 2024-09-21T12:37:47.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:46 smithi078 ceph-mon[63866]: pgmap v323: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 6.1 MiB/s wr, 155 op/s 2024-09-21T12:37:47.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:46 smithi078 ceph-mon[63866]: Upgrade: It appears safe to stop mds.foofs.smithi148.uukezr 2024-09-21T12:37:48.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:48 smithi148 ceph-mon[49387]: Upgrade: Updating mds.foofs.smithi148.uukezr 2024-09-21T12:37:48.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:48 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:48.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:48 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi148.uukezr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T12:37:48.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:48 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:37:48.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:48 smithi148 ceph-mon[49387]: Deploying daemon mds.foofs.smithi148.uukezr on smithi148 2024-09-21T12:37:48.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:48 smithi148 ceph-mon[49387]: pgmap v324: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 4.6 MiB/s wr, 129 op/s 2024-09-21T12:37:48.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:48 smithi078 ceph-mon[63866]: Upgrade: Updating mds.foofs.smithi148.uukezr 2024-09-21T12:37:48.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:48 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:48.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:48 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi148.uukezr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T12:37:48.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:48 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:37:48.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:48 smithi078 ceph-mon[63866]: Deploying daemon mds.foofs.smithi148.uukezr on smithi148 2024-09-21T12:37:48.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:48 smithi078 ceph-mon[63866]: pgmap v324: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 4.6 MiB/s wr, 129 op/s 2024-09-21T12:37:49.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:49 smithi148 ceph-mon[49387]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-21T12:37:49.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:49 smithi148 ceph-mon[49387]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T12:37:49.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:49 smithi148 ceph-mon[49387]: osdmap e160: 8 total, 8 up, 8 in 2024-09-21T12:37:49.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:49 smithi148 ceph-mon[49387]: Standby daemon mds.foofs.smithi078.ytpzwh assigned to filesystem foofs as rank 0 2024-09-21T12:37:49.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:49 smithi148 ceph-mon[49387]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-21T12:37:49.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:49 smithi148 ceph-mon[49387]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T12:37:49.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:49 smithi148 ceph-mon[49387]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-21T12:37:49.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:49 smithi148 ceph-mon[49387]: fsmap foofs:1/1 {0=foofs.smithi078.ytpzwh=up:replay} 2024-09-21T12:37:49.302 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:37:48 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:37:48.985+0000 7f83fc270640 -1 bdev(0x55d3577e2c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 5 2024-09-21T12:37:49.302 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:37:48 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:37:48.986+0000 7f83faa6d640 -1 bdev(0x55d3577e2c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 5 2024-09-21T12:37:49.302 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:37:48 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:37:48.986+0000 7f83fb26e640 -1 bdev(0x55d3577e2c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 5 2024-09-21T12:37:49.302 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:37:48 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:37:48.980+0000 7fb98be0e640 -1 bdev(0x56269fc36c00 /var/lib/ceph/osd/ceph-7/block) aio_submit retries 1 2024-09-21T12:37:49.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:48 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi078[63862]: 2024-09-21T12:37:48.910+0000 7f79f2431640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T12:37:49.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:49 smithi078 ceph-mon[63866]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-21T12:37:49.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:49 smithi078 ceph-mon[63866]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T12:37:49.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:49 smithi078 ceph-mon[63866]: osdmap e160: 8 total, 8 up, 8 in 2024-09-21T12:37:49.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:49 smithi078 ceph-mon[63866]: Standby daemon mds.foofs.smithi078.ytpzwh assigned to filesystem foofs as rank 0 2024-09-21T12:37:49.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:49 smithi078 ceph-mon[63866]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-21T12:37:49.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:49 smithi078 ceph-mon[63866]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T12:37:49.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:49 smithi078 ceph-mon[63866]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-21T12:37:49.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:49 smithi078 ceph-mon[63866]: fsmap foofs:1/1 {0=foofs.smithi078.ytpzwh=up:replay} 2024-09-21T12:37:50.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:50 smithi148 ceph-mon[49387]: pgmap v326: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 201 KiB/s wr, 30 op/s 2024-09-21T12:37:50.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:50 smithi078 ceph-mon[63866]: pgmap v326: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 201 KiB/s wr, 30 op/s 2024-09-21T12:37:51.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:51 smithi148 ceph-mon[49387]: reconnect by client.14556 172.21.15.78:0/1840377618 after 0 2024-09-21T12:37:51.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:51 smithi148 ceph-mon[49387]: mds.? [v2:172.21.15.78:6834/4075310113,v1:172.21.15.78:6835/4075310113] up:reconnect 2024-09-21T12:37:51.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:51 smithi148 ceph-mon[49387]: fsmap foofs:1/1 {0=foofs.smithi078.ytpzwh=up:reconnect} 2024-09-21T12:37:51.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:51 smithi148 ceph-mon[49387]: reconnect by client.24327 172.21.15.148:0/3482514271 after 0.0229996 2024-09-21T12:37:51.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:51 smithi078 ceph-mon[63866]: reconnect by client.14556 172.21.15.78:0/1840377618 after 0 2024-09-21T12:37:51.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:51 smithi078 ceph-mon[63866]: mds.? [v2:172.21.15.78:6834/4075310113,v1:172.21.15.78:6835/4075310113] up:reconnect 2024-09-21T12:37:51.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:51 smithi078 ceph-mon[63866]: fsmap foofs:1/1 {0=foofs.smithi078.ytpzwh=up:reconnect} 2024-09-21T12:37:51.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:51 smithi078 ceph-mon[63866]: reconnect by client.24327 172.21.15.148:0/3482514271 after 0.0229996 2024-09-21T12:37:52.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:52 smithi148 ceph-mon[49387]: mds.? [v2:172.21.15.78:6834/4075310113,v1:172.21.15.78:6835/4075310113] up:rejoin 2024-09-21T12:37:52.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:52 smithi148 ceph-mon[49387]: fsmap foofs:1/1 {0=foofs.smithi078.ytpzwh=up:rejoin} 2024-09-21T12:37:52.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:52 smithi148 ceph-mon[49387]: daemon mds.foofs.smithi078.ytpzwh is now active in filesystem foofs as rank 0 2024-09-21T12:37:52.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:52 smithi148 ceph-mon[49387]: pgmap v327: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 71 KiB/s wr, 23 op/s 2024-09-21T12:37:52.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:52 smithi078 ceph-mon[63866]: mds.? [v2:172.21.15.78:6834/4075310113,v1:172.21.15.78:6835/4075310113] up:rejoin 2024-09-21T12:37:52.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:52 smithi078 ceph-mon[63866]: fsmap foofs:1/1 {0=foofs.smithi078.ytpzwh=up:rejoin} 2024-09-21T12:37:52.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:52 smithi078 ceph-mon[63866]: daemon mds.foofs.smithi078.ytpzwh is now active in filesystem foofs as rank 0 2024-09-21T12:37:52.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:52 smithi078 ceph-mon[63866]: pgmap v327: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 71 KiB/s wr, 23 op/s 2024-09-21T12:37:53.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:53 smithi148 ceph-mon[49387]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-21T12:37:53.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:53 smithi148 ceph-mon[49387]: mds.? [v2:172.21.15.78:6834/4075310113,v1:172.21.15.78:6835/4075310113] up:active 2024-09-21T12:37:53.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:53 smithi148 ceph-mon[49387]: fsmap foofs:1 {0=foofs.smithi078.ytpzwh=up:active} 2024-09-21T12:37:53.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:53 smithi078 ceph-mon[63866]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-21T12:37:53.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:53 smithi078 ceph-mon[63866]: mds.? [v2:172.21.15.78:6834/4075310113,v1:172.21.15.78:6835/4075310113] up:active 2024-09-21T12:37:53.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:53 smithi078 ceph-mon[63866]: fsmap foofs:1 {0=foofs.smithi078.ytpzwh=up:active} 2024-09-21T12:37:54.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:54 smithi148 ceph-mon[49387]: pgmap v328: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 19 KiB/s wr, 11 op/s 2024-09-21T12:37:54.555 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:54 smithi078 ceph-mon[63866]: pgmap v328: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 19 KiB/s wr, 11 op/s 2024-09-21T12:37:56.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:56 smithi148 ceph-mon[49387]: pgmap v329: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 19 KiB/s wr, 11 op/s 2024-09-21T12:37:57.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:56 smithi078 ceph-mon[63866]: pgmap v329: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 19 KiB/s wr, 11 op/s 2024-09-21T12:37:58.579 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-21T12:37:59.213 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:58 smithi148 ceph-mon[49387]: pgmap v330: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 8.8 KiB/s wr, 15 op/s 2024-09-21T12:37:59.213 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:58 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:59.213 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:58 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:59.213 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:58 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:37:59.213 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:58 smithi148 ceph-mon[49387]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-21T12:37:59.213 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:58 smithi148 ceph-mon[49387]: Cluster is now healthy 2024-09-21T12:37:59.213 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:58 smithi148 ceph-mon[49387]: mds.? [v2:172.21.15.148:6832/2265773494,v1:172.21.15.148:6833/2265773494] up:boot 2024-09-21T12:37:59.214 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:58 smithi148 ceph-mon[49387]: fsmap foofs:1 {0=foofs.smithi078.ytpzwh=up:active} 1 up:standby 2024-09-21T12:37:59.214 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:58 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mds metadata", "who": "foofs.smithi148.uukezr"} : dispatch 2024-09-21T12:37:59.284 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T12:37:59.285 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (10m) 18s ago 19m 20.1M - 0.27.0 11f11916f8cd e6e67ab07dec 2024-09-21T12:37:59.285 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (6m) 18s ago 19m 6895k - 19.3.0-5104-g44178494 b415816db8b4 46a10ea0b111 2024-09-21T12:37:59.285 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi148 smithi148 running (6m) 46s ago 18m 6895k - 19.3.0-5104-g44178494 b415816db8b4 23deb0615621 2024-09-21T12:37:59.285 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (10m) 18s ago 18m 53.5M - dad864ee21e9 2a140c36091b 2024-09-21T12:37:59.285 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.ytpzwh smithi078 running (23s) 18s ago 13m 10.8M - 19.3.0-5104-g44178494 b415816db8b4 65fd79480937 2024-09-21T12:37:59.285 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi148.uukezr smithi148 starting - - - - 2024-09-21T12:37:59.286 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.ntwyan smithi078 *:8443,9283,8765 running (9m) 18s ago 20m 523M - 19.3.0-5104-g44178494 b415816db8b4 a61baedebf43 2024-09-21T12:37:59.286 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi148.oyiqmx smithi148 *:8443,9283,8765 running (8m) 46s ago 18m 458M - 19.3.0-5104-g44178494 b415816db8b4 0255535c6fdb 2024-09-21T12:37:59.286 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (8m) 18s ago 20m 65.8M 2048M 19.3.0-5104-g44178494 b415816db8b4 22b3097efbd8 2024-09-21T12:37:59.286 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi148 smithi148 running (7m) 46s ago 18m 46.6M 2048M 19.3.0-5104-g44178494 b415816db8b4 df8b08509136 2024-09-21T12:37:59.286 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.ydbhru smithi078 *:2049 running (12m) 18s ago 12m 332M - 4.0 e1d6a67b021e 7e5b0274ac38 2024-09-21T12:37:59.286 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi148.einuzt smithi148 *:2049 running (12m) 46s ago 12m 87.3M - 4.0 e1d6a67b021e 76a8b0eabede 2024-09-21T12:37:59.286 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (9m) 18s ago 18m 11.3M - 1.7.0 72c9c2088986 301adb4e9e22 2024-09-21T12:37:59.287 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi148 smithi148 *:9100 running (9m) 46s ago 18m 12.3M - 1.7.0 72c9c2088986 0cceff4bf342 2024-09-21T12:37:59.287 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (6m) 18s ago 17m 411M 1936M 19.3.0-5104-g44178494 b415816db8b4 7374add97534 2024-09-21T12:37:59.287 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (5m) 18s ago 17m 313M 1936M 19.3.0-5104-g44178494 b415816db8b4 14655a8da4e5 2024-09-21T12:37:59.287 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (4m) 18s ago 16m 294M 1936M 19.3.0-5104-g44178494 b415816db8b4 07ceda32f383 2024-09-21T12:37:59.287 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (3m) 18s ago 16m 380M 1936M 19.3.0-5104-g44178494 b415816db8b4 ead87e4306ff 2024-09-21T12:37:59.287 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi148 running (2m) 46s ago 15m 331M 2704M 19.3.0-5104-g44178494 b415816db8b4 10ec7d6b476e 2024-09-21T12:37:59.287 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi148 running (2m) 46s ago 15m 298M 2704M 19.3.0-5104-g44178494 b415816db8b4 06ac02700ea1 2024-09-21T12:37:59.288 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi148 running (89s) 46s ago 15m 246M 2704M 19.3.0-5104-g44178494 b415816db8b4 17613684a653 2024-09-21T12:37:59.288 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi148 running (51s) 46s ago 14m 14.4M 2704M 19.3.0-5104-g44178494 b415816db8b4 2e6a6f842c47 2024-09-21T12:37:59.288 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (8m) 18s ago 18m 56.0M - 2.51.0 1d3b7f56885b face2d4cdd1b 2024-09-21T12:37:59.288 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:58 smithi078 ceph-mon[63866]: pgmap v330: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 8.8 KiB/s wr, 15 op/s 2024-09-21T12:37:59.288 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:58 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:59.288 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:58 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:37:59.288 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:58 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:37:59.289 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:58 smithi078 ceph-mon[63866]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-21T12:37:59.289 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:58 smithi078 ceph-mon[63866]: Cluster is now healthy 2024-09-21T12:37:59.289 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:58 smithi078 ceph-mon[63866]: mds.? [v2:172.21.15.148:6832/2265773494,v1:172.21.15.148:6833/2265773494] up:boot 2024-09-21T12:37:59.289 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:58 smithi078 ceph-mon[63866]: fsmap foofs:1 {0=foofs.smithi078.ytpzwh=up:active} 1 up:standby 2024-09-21T12:37:59.289 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:58 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "mds metadata", "who": "foofs.smithi148.uukezr"} : dispatch 2024-09-21T12:37:59.716 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:37:59.717 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-21T12:37:59.717 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:37:59.717 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:37:59.717 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-21T12:37:59.717 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:37:59.717 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:37:59.717 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-21T12:37:59.717 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 8 2024-09-21T12:37:59.717 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:37:59.717 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-21T12:37:59.717 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:37:59.717 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:37:59.717 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-21T12:37:59.718 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 14 2024-09-21T12:37:59.718 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T12:37:59.718 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:38:00.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:59 smithi078 ceph-mon[63866]: from='client.34482 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:38:00.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:59 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/3173814086' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:00.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:37:59 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:38:00.076 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:38:00.076 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5", 2024-09-21T12:38:00.076 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-21T12:38:00.076 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T12:38:00.076 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-21T12:38:00.076 INFO:teuthology.orchestra.run.smithi078.stdout: "crash", 2024-09-21T12:38:00.076 INFO:teuthology.orchestra.run.smithi078.stdout: "osd", 2024-09-21T12:38:00.076 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-09-21T12:38:00.077 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr" 2024-09-21T12:38:00.077 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-21T12:38:00.077 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "15/23 daemons upgraded", 2024-09-21T12:38:00.077 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading mds daemons", 2024-09-21T12:38:00.077 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-21T12:38:00.077 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:38:00.165 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:59 smithi148 ceph-mon[49387]: from='client.34482 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:38:00.166 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:59 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/3173814086' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:00.166 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:37:59 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:38:00.499 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-21T12:38:01.120 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:00 smithi148 ceph-mon[49387]: from='client.34486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:38:01.120 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:00 smithi148 ceph-mon[49387]: from='client.34490 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:38:01.120 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:00 smithi148 ceph-mon[49387]: pgmap v331: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 8.3 KiB/s wr, 14 op/s 2024-09-21T12:38:01.120 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:00 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/4183018183' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:38:01.120 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:01.120 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:01.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:00 smithi078 ceph-mon[63866]: from='client.34486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:38:01.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:00 smithi078 ceph-mon[63866]: from='client.34490 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:38:01.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:00 smithi078 ceph-mon[63866]: pgmap v331: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 8.3 KiB/s wr, 14 op/s 2024-09-21T12:38:01.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:00 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/4183018183' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:38:01.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:01.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:01.996 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:01 smithi148 ceph-mon[49387]: from='client.34498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:38:01.996 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:01 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:01.996 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:01 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:02.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:01 smithi078 ceph-mon[63866]: from='client.34498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:38:02.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:01 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:02.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:01 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:03.127 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:02 smithi148 ceph-mon[49387]: pgmap v332: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 7.3 KiB/s wr, 12 op/s 2024-09-21T12:38:03.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:02 smithi078 ceph-mon[63866]: pgmap v332: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 7.3 KiB/s wr, 12 op/s 2024-09-21T12:38:05.197 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:04 smithi078 ceph-mon[63866]: pgmap v333: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 7.3 KiB/s wr, 10 op/s 2024-09-21T12:38:05.198 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:04 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:05.198 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:04 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:05.198 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:04 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:38:05.198 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:04 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:38:05.198 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:04 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:05.198 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:04 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:05.198 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:04 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:38:05.198 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:04 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:05.198 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:04 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:05.198 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:04 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:05.198 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:04 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:05.199 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:04 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:05.199 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:04 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:05.199 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:04 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi078.ytpzwh"} : dispatch 2024-09-21T12:38:05.199 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:04 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi078.ytpzwh"}]': finished 2024-09-21T12:38:05.199 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:04 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi148.uukezr"} : dispatch 2024-09-21T12:38:05.199 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:04 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi148.uukezr"}]': finished 2024-09-21T12:38:05.199 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:04 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:05.199 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:04 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:05.199 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:04 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:05.199 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:04 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:05.199 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:04 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:05.199 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:04 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:05.199 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:04 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:05.199 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:04 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:05.200 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:04 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:05.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:04 smithi148 ceph-mon[49387]: pgmap v333: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 7.3 KiB/s wr, 10 op/s 2024-09-21T12:38:05.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:04 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:05.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:04 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:05.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:04 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:38:05.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:04 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:38:05.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:04 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:05.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:04 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:05.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:04 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:38:05.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:04 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:05.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:04 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:05.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:04 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:05.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:04 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:05.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:04 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:05.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:04 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:05.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:04 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi078.ytpzwh"} : dispatch 2024-09-21T12:38:05.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:04 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi078.ytpzwh"}]': finished 2024-09-21T12:38:05.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:04 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi148.uukezr"} : dispatch 2024-09-21T12:38:05.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:04 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi148.uukezr"}]': finished 2024-09-21T12:38:05.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:04 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:05.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:04 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:05.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:04 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:05.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:04 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:05.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:04 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:05.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:04 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:05.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:04 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:05.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:04 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:05.302 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:04 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:06.246 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:05 smithi078 ceph-mon[63866]: Detected new or changed devices on smithi148 2024-09-21T12:38:06.246 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:05 smithi078 ceph-mon[63866]: Upgrade: Setting container_image for all mds 2024-09-21T12:38:06.246 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:05 smithi078 ceph-mon[63866]: Upgrade: Setting container_image for all rgw 2024-09-21T12:38:06.246 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:05 smithi078 ceph-mon[63866]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T12:38:06.247 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:05 smithi078 ceph-mon[63866]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T12:38:06.247 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:05 smithi078 ceph-mon[63866]: Upgrade: Setting container_image for all iscsi 2024-09-21T12:38:06.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:05 smithi148 ceph-mon[49387]: Detected new or changed devices on smithi148 2024-09-21T12:38:06.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:05 smithi148 ceph-mon[49387]: Upgrade: Setting container_image for all mds 2024-09-21T12:38:06.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:05 smithi148 ceph-mon[49387]: Upgrade: Setting container_image for all rgw 2024-09-21T12:38:06.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:05 smithi148 ceph-mon[49387]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T12:38:06.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:05 smithi148 ceph-mon[49387]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T12:38:06.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:05 smithi148 ceph-mon[49387]: Upgrade: Setting container_image for all iscsi 2024-09-21T12:38:07.202 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:06 smithi148 ceph-mon[49387]: pgmap v334: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.0 KiB/s rd, 7.2 KiB/s wr, 4 op/s 2024-09-21T12:38:07.202 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:06 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:07.202 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:06 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.ydbhru", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-21T12:38:07.202 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:06 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-21T12:38:07.202 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:06 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T12:38:07.202 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:06 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:38:07.203 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:06 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-21T12:38:07.203 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:06 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T12:38:07.203 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:06 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.ydbhru-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-21T12:38:07.203 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:06 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:38:07.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:06 smithi078 ceph-mon[63866]: pgmap v334: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.0 KiB/s rd, 7.2 KiB/s wr, 4 op/s 2024-09-21T12:38:07.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:06 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:07.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:06 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.ydbhru", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-21T12:38:07.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:06 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-21T12:38:07.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:06 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T12:38:07.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:06 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:38:07.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:06 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-21T12:38:07.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:06 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T12:38:07.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:06 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi078.ydbhru-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-21T12:38:07.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:06 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:38:08.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:07 smithi078 ceph-mon[63866]: Upgrade: Updating nfs.foo.0.0.smithi078.ydbhru (1/2) 2024-09-21T12:38:08.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:07 smithi078 ceph-mon[63866]: Creating key for client.nfs.foo.0.0.smithi078.ydbhru 2024-09-21T12:38:08.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:07 smithi078 ceph-mon[63866]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-21T12:38:08.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:07 smithi078 ceph-mon[63866]: Rados config object exists: conf-nfs.foo 2024-09-21T12:38:08.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:07 smithi078 ceph-mon[63866]: Creating key for client.nfs.foo.0.0.smithi078.ydbhru-rgw 2024-09-21T12:38:08.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:07 smithi078 ceph-mon[63866]: Bind address in nfs.foo.0.0.smithi078.ydbhru's ganesha conf is defaulting to empty 2024-09-21T12:38:08.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:07 smithi078 ceph-mon[63866]: Deploying daemon nfs.foo.0.0.smithi078.ydbhru on smithi078 2024-09-21T12:38:08.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:07 smithi078 ceph-mon[63866]: pgmap v335: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.3 KiB/s rd, 7.3 KiB/s wr, 5 op/s 2024-09-21T12:38:08.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:07 smithi148 ceph-mon[49387]: Upgrade: Updating nfs.foo.0.0.smithi078.ydbhru (1/2) 2024-09-21T12:38:08.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:07 smithi148 ceph-mon[49387]: Creating key for client.nfs.foo.0.0.smithi078.ydbhru 2024-09-21T12:38:08.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:07 smithi148 ceph-mon[49387]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-21T12:38:08.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:07 smithi148 ceph-mon[49387]: Rados config object exists: conf-nfs.foo 2024-09-21T12:38:08.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:07 smithi148 ceph-mon[49387]: Creating key for client.nfs.foo.0.0.smithi078.ydbhru-rgw 2024-09-21T12:38:08.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:07 smithi148 ceph-mon[49387]: Bind address in nfs.foo.0.0.smithi078.ydbhru's ganesha conf is defaulting to empty 2024-09-21T12:38:08.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:07 smithi148 ceph-mon[49387]: Deploying daemon nfs.foo.0.0.smithi078.ydbhru on smithi078 2024-09-21T12:38:08.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:07 smithi148 ceph-mon[49387]: pgmap v335: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.3 KiB/s rd, 7.3 KiB/s wr, 5 op/s 2024-09-21T12:38:10.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:10 smithi148 ceph-mon[49387]: pgmap v336: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s 2024-09-21T12:38:10.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:10 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:10.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:10 smithi148 ceph-mon[49387]: from='mds.? [v2:172.21.15.78:6834/4075310113,v1:172.21.15.78:6835/4075310113]' entity='mds.foofs.smithi078.ytpzwh' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.78:0/1840377618"} : dispatch 2024-09-21T12:38:10.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:10 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:11.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:10 smithi078 ceph-mon[63866]: pgmap v336: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s 2024-09-21T12:38:11.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:10 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:11.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:10 smithi078 ceph-mon[63866]: from='mds.? [v2:172.21.15.78:6834/4075310113,v1:172.21.15.78:6835/4075310113]' entity='mds.foofs.smithi078.ytpzwh' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.78:0/1840377618"} : dispatch 2024-09-21T12:38:11.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:10 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:11.963 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:11 smithi148 ceph-mon[49387]: Evicting (and blocklisting) client session 14556 (172.21.15.78:0/1840377618) 2024-09-21T12:38:11.963 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:11 smithi148 ceph-mon[49387]: from='mds.? [v2:172.21.15.78:6834/4075310113,v1:172.21.15.78:6835/4075310113]' entity='mds.foofs.smithi078.ytpzwh' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.78:0/1840377618"}]': finished 2024-09-21T12:38:11.963 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:11 smithi148 ceph-mon[49387]: osdmap e161: 8 total, 8 up, 8 in 2024-09-21T12:38:12.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:11 smithi078 ceph-mon[63866]: Evicting (and blocklisting) client session 14556 (172.21.15.78:0/1840377618) 2024-09-21T12:38:12.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:11 smithi078 ceph-mon[63866]: from='mds.? [v2:172.21.15.78:6834/4075310113,v1:172.21.15.78:6835/4075310113]' entity='mds.foofs.smithi078.ytpzwh' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.78:0/1840377618"}]': finished 2024-09-21T12:38:12.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:11 smithi078 ceph-mon[63866]: osdmap e161: 8 total, 8 up, 8 in 2024-09-21T12:38:12.700 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:12 smithi148 ceph-mon[49387]: pgmap v337: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s 2024-09-21T12:38:12.700 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:12 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:12.701 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:12 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi148.einuzt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-21T12:38:12.701 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:12 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-21T12:38:12.701 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:12 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T12:38:12.701 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:12 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:38:12.701 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:12 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-21T12:38:12.701 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:12 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T12:38:12.701 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:12 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi148.einuzt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-21T12:38:12.701 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:12 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:38:13.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:12 smithi078 ceph-mon[63866]: pgmap v337: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s 2024-09-21T12:38:13.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:12 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:13.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:12 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi148.einuzt", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-21T12:38:13.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:12 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-21T12:38:13.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:12 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-21T12:38:13.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:12 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:38:13.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:12 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-21T12:38:13.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:12 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-21T12:38:13.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:12 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi148.einuzt-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-21T12:38:13.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:12 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:38:13.756 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:13 smithi148 ceph-mon[49387]: Upgrade: Updating nfs.foo.1.0.smithi148.einuzt (2/2) 2024-09-21T12:38:13.757 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:13 smithi148 ceph-mon[49387]: Creating key for client.nfs.foo.1.0.smithi148.einuzt 2024-09-21T12:38:13.757 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:13 smithi148 ceph-mon[49387]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-21T12:38:13.757 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:13 smithi148 ceph-mon[49387]: Rados config object exists: conf-nfs.foo 2024-09-21T12:38:13.757 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:13 smithi148 ceph-mon[49387]: Creating key for client.nfs.foo.1.0.smithi148.einuzt-rgw 2024-09-21T12:38:13.757 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:13 smithi148 ceph-mon[49387]: Bind address in nfs.foo.1.0.smithi148.einuzt's ganesha conf is defaulting to empty 2024-09-21T12:38:13.757 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:13 smithi148 ceph-mon[49387]: Deploying daemon nfs.foo.1.0.smithi148.einuzt on smithi148 2024-09-21T12:38:14.047 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:13 smithi078 ceph-mon[63866]: Upgrade: Updating nfs.foo.1.0.smithi148.einuzt (2/2) 2024-09-21T12:38:14.047 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:13 smithi078 ceph-mon[63866]: Creating key for client.nfs.foo.1.0.smithi148.einuzt 2024-09-21T12:38:14.047 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:13 smithi078 ceph-mon[63866]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-21T12:38:14.047 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:13 smithi078 ceph-mon[63866]: Rados config object exists: conf-nfs.foo 2024-09-21T12:38:14.047 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:13 smithi078 ceph-mon[63866]: Creating key for client.nfs.foo.1.0.smithi148.einuzt-rgw 2024-09-21T12:38:14.047 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:13 smithi078 ceph-mon[63866]: Bind address in nfs.foo.1.0.smithi148.einuzt's ganesha conf is defaulting to empty 2024-09-21T12:38:14.048 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:13 smithi078 ceph-mon[63866]: Deploying daemon nfs.foo.1.0.smithi148.einuzt on smithi148 2024-09-21T12:38:15.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:14 smithi148 ceph-mon[49387]: pgmap v339: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-09-21T12:38:15.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:14 smithi078 ceph-mon[63866]: pgmap v339: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-09-21T12:38:16.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:15 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:38:16.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:15 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:38:17.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:16 smithi148 ceph-mon[49387]: pgmap v340: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-09-21T12:38:17.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:16 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:17.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:16 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:17.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:16 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:38:17.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:16 smithi078 ceph-mon[63866]: pgmap v340: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-09-21T12:38:17.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:16 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:17.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:16 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:17.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:16 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:38:19.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:18 smithi148 ceph-mon[49387]: pgmap v341: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.0 KiB/s wr, 7 op/s 2024-09-21T12:38:19.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:18 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:19.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:18 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:19.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:18 smithi078 ceph-mon[63866]: pgmap v341: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.0 KiB/s wr, 7 op/s 2024-09-21T12:38:19.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:18 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:19.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:18 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:19.931 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:19 smithi148 ceph-mon[49387]: from='mds.? [v2:172.21.15.78:6834/4075310113,v1:172.21.15.78:6835/4075310113]' entity='mds.foofs.smithi078.ytpzwh' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.148:0/3482514271"} : dispatch 2024-09-21T12:38:19.931 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:19 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:19.931 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:19 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:20.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:19 smithi078 ceph-mon[63866]: from='mds.? [v2:172.21.15.78:6834/4075310113,v1:172.21.15.78:6835/4075310113]' entity='mds.foofs.smithi078.ytpzwh' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.148:0/3482514271"} : dispatch 2024-09-21T12:38:20.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:19 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:20.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:19 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:21.048 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:20 smithi148 ceph-mon[49387]: Evicting (and blocklisting) client session 24327 (172.21.15.148:0/3482514271) 2024-09-21T12:38:21.048 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:20 smithi148 ceph-mon[49387]: pgmap v342: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.0 KiB/s wr, 7 op/s 2024-09-21T12:38:21.048 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:20 smithi148 ceph-mon[49387]: from='mds.? [v2:172.21.15.78:6834/4075310113,v1:172.21.15.78:6835/4075310113]' entity='mds.foofs.smithi078.ytpzwh' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.148:0/3482514271"}]': finished 2024-09-21T12:38:21.048 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:20 smithi148 ceph-mon[49387]: osdmap e162: 8 total, 8 up, 8 in 2024-09-21T12:38:21.049 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:20 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:21.049 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:20 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:21.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:20 smithi078 ceph-mon[63866]: Evicting (and blocklisting) client session 24327 (172.21.15.148:0/3482514271) 2024-09-21T12:38:21.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:20 smithi078 ceph-mon[63866]: pgmap v342: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.0 KiB/s wr, 7 op/s 2024-09-21T12:38:21.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:20 smithi078 ceph-mon[63866]: from='mds.? [v2:172.21.15.78:6834/4075310113,v1:172.21.15.78:6835/4075310113]' entity='mds.foofs.smithi078.ytpzwh' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.148:0/3482514271"}]': finished 2024-09-21T12:38:21.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:20 smithi078 ceph-mon[63866]: osdmap e162: 8 total, 8 up, 8 in 2024-09-21T12:38:21.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:20 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:21.307 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:20 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:22.489 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:22 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:22.490 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:22 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:22.490 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:22 smithi148 ceph-mon[49387]: pgmap v344: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.0 KiB/s wr, 7 op/s 2024-09-21T12:38:22.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:22 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:22.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:22 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:22.556 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:22 smithi078 ceph-mon[63866]: pgmap v344: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.0 KiB/s wr, 7 op/s 2024-09-21T12:38:23.778 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:23 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:23.779 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:23 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:23.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:23 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:23.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:23 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:24.730 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:24 smithi078 ceph-mon[63866]: pgmap v345: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-21T12:38:24.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:24 smithi148 ceph-mon[49387]: pgmap v345: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-21T12:38:26.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:25 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:26.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:25 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:26.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:25 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:38:26.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:25 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:38:26.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:25 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:26.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:25 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:26.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:25 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:38:26.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:25 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:26.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:25 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:26.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:25 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:26.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:25 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:26.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:25 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:26.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:25 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:26.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:25 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:26.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:25 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:26.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:25 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:26.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:25 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:26.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:25 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:26.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:25 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:26.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:25 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi078.ydbhru"} : dispatch 2024-09-21T12:38:26.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:25 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi078.ydbhru"}]': finished 2024-09-21T12:38:26.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:25 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi148.einuzt"} : dispatch 2024-09-21T12:38:26.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:25 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi148.einuzt"}]': finished 2024-09-21T12:38:26.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:25 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:26.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:25 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:26.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:25 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:26.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:25 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:26.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:25 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:26.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:25 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:26.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:25 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:26.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:25 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:38:26.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:25 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:38:26.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:25 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:26.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:25 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:26.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:25 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:38:26.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:25 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:26.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:25 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:26.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:25 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:26.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:25 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:26.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:25 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:26.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:25 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:26.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:25 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:26.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:25 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:26.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:25 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:26.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:25 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:26.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:25 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:26.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:25 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:26.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:25 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi078.ydbhru"} : dispatch 2024-09-21T12:38:26.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:25 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi078.ydbhru"}]': finished 2024-09-21T12:38:26.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:25 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi148.einuzt"} : dispatch 2024-09-21T12:38:26.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:25 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi148.einuzt"}]': finished 2024-09-21T12:38:26.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:25 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:26.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:25 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:26.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:25 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:26.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:25 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:26.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:25 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:27.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:26 smithi148 ceph-mon[49387]: Upgrade: Setting container_image for all nfs 2024-09-21T12:38:27.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:26 smithi148 ceph-mon[49387]: Upgrade: Setting container_image for all nvmeof 2024-09-21T12:38:27.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:26 smithi148 ceph-mon[49387]: pgmap v346: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-21T12:38:27.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:26 smithi078 ceph-mon[63866]: Upgrade: Setting container_image for all nfs 2024-09-21T12:38:27.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:26 smithi078 ceph-mon[63866]: Upgrade: Setting container_image for all nvmeof 2024-09-21T12:38:27.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:26 smithi078 ceph-mon[63866]: pgmap v346: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-21T12:38:28.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:27 smithi148 ceph-mon[49387]: Upgrade: Updating grafana.smithi078 2024-09-21T12:38:28.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:27 smithi148 ceph-mon[49387]: Deploying daemon grafana.smithi078 on smithi078 2024-09-21T12:38:28.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:27 smithi078 ceph-mon[63866]: Upgrade: Updating grafana.smithi078 2024-09-21T12:38:28.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:27 smithi078 ceph-mon[63866]: Deploying daemon grafana.smithi078 on smithi078 2024-09-21T12:38:29.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:28 smithi148 ceph-mon[49387]: pgmap v347: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 102 B/s rd, 409 B/s wr, 0 op/s 2024-09-21T12:38:29.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:28 smithi078 ceph-mon[63866]: pgmap v347: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 102 B/s rd, 409 B/s wr, 0 op/s 2024-09-21T12:38:30.927 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-21T12:38:31.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:30 smithi148 ceph-mon[49387]: pgmap v348: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 102 B/s rd, 409 B/s wr, 0 op/s 2024-09-21T12:38:31.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:38:31.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:30 smithi078 ceph-mon[63866]: pgmap v348: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 102 B/s rd, 409 B/s wr, 0 op/s 2024-09-21T12:38:31.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:38:31.688 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T12:38:31.689 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (10m) 11s ago 19m 20.2M - 0.27.0 11f11916f8cd e6e67ab07dec 2024-09-21T12:38:31.689 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (7m) 11s ago 19m 6895k - 19.3.0-5104-g44178494 b415816db8b4 46a10ea0b111 2024-09-21T12:38:31.689 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi148 smithi148 running (7m) 12s ago 19m 6895k - 19.3.0-5104-g44178494 b415816db8b4 23deb0615621 2024-09-21T12:38:31.689 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (10m) 11s ago 19m 54.7M - dad864ee21e9 2a140c36091b 2024-09-21T12:38:31.689 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.ytpzwh smithi078 running (56s) 11s ago 13m 29.4M - 19.3.0-5104-g44178494 b415816db8b4 65fd79480937 2024-09-21T12:38:31.689 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi148.uukezr smithi148 running (34s) 12s ago 13m 12.4M - 19.3.0-5104-g44178494 b415816db8b4 3e9d00854170 2024-09-21T12:38:31.689 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.ntwyan smithi078 *:8443,9283,8765 running (9m) 11s ago 20m 525M - 19.3.0-5104-g44178494 b415816db8b4 a61baedebf43 2024-09-21T12:38:31.689 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi148.oyiqmx smithi148 *:8443,9283,8765 running (8m) 12s ago 19m 458M - 19.3.0-5104-g44178494 b415816db8b4 0255535c6fdb 2024-09-21T12:38:31.689 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (8m) 11s ago 21m 67.4M 2048M 19.3.0-5104-g44178494 b415816db8b4 22b3097efbd8 2024-09-21T12:38:31.689 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi148 smithi148 running (8m) 12s ago 18m 49.5M 2048M 19.3.0-5104-g44178494 b415816db8b4 df8b08509136 2024-09-21T12:38:31.689 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.ydbhru smithi078 *:2049 running (21s) 11s ago 13m 66.4M - 5.9 b415816db8b4 4c82527787a5 2024-09-21T12:38:31.689 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi148.einuzt smithi148 *:2049 running (16s) 12s ago 13m 17.7M - 5.9 b415816db8b4 326968a99b8f 2024-09-21T12:38:31.689 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (10m) 11s ago 19m 10.9M - 1.7.0 72c9c2088986 301adb4e9e22 2024-09-21T12:38:31.690 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi148 smithi148 *:9100 running (9m) 12s ago 18m 11.9M - 1.7.0 72c9c2088986 0cceff4bf342 2024-09-21T12:38:31.690 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (6m) 11s ago 18m 411M 1936M 19.3.0-5104-g44178494 b415816db8b4 7374add97534 2024-09-21T12:38:31.690 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (5m) 11s ago 17m 315M 1936M 19.3.0-5104-g44178494 b415816db8b4 14655a8da4e5 2024-09-21T12:38:31.690 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (5m) 11s ago 17m 295M 1936M 19.3.0-5104-g44178494 b415816db8b4 07ceda32f383 2024-09-21T12:38:31.690 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (4m) 11s ago 16m 383M 1936M 19.3.0-5104-g44178494 b415816db8b4 ead87e4306ff 2024-09-21T12:38:31.690 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi148 running (3m) 12s ago 16m 338M 2704M 19.3.0-5104-g44178494 b415816db8b4 10ec7d6b476e 2024-09-21T12:38:31.690 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi148 running (2m) 12s ago 16m 326M 2704M 19.3.0-5104-g44178494 b415816db8b4 06ac02700ea1 2024-09-21T12:38:31.690 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi148 running (2m) 12s ago 15m 280M 2704M 19.3.0-5104-g44178494 b415816db8b4 17613684a653 2024-09-21T12:38:31.690 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi148 running (83s) 12s ago 15m 299M 2704M 19.3.0-5104-g44178494 b415816db8b4 2e6a6f842c47 2024-09-21T12:38:31.690 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (9m) 11s ago 18m 56.0M - 2.51.0 1d3b7f56885b face2d4cdd1b 2024-09-21T12:38:32.359 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:38:32.359 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-21T12:38:32.359 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:38:32.360 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:38:32.360 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-21T12:38:32.360 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:38:32.360 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:38:32.360 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-21T12:38:32.360 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 8 2024-09-21T12:38:32.360 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:38:32.361 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-21T12:38:32.361 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:38:32.361 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:38:32.361 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-21T12:38:32.361 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 14 2024-09-21T12:38:32.361 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T12:38:32.362 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:38:32.849 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:38:32.849 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5", 2024-09-21T12:38:32.849 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": true, 2024-09-21T12:38:32.849 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T12:38:32.849 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [ 2024-09-21T12:38:32.849 INFO:teuthology.orchestra.run.smithi078.stdout: "mds", 2024-09-21T12:38:32.849 INFO:teuthology.orchestra.run.smithi078.stdout: "mon", 2024-09-21T12:38:32.849 INFO:teuthology.orchestra.run.smithi078.stdout: "osd", 2024-09-21T12:38:32.849 INFO:teuthology.orchestra.run.smithi078.stdout: "nfs", 2024-09-21T12:38:32.849 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr", 2024-09-21T12:38:32.849 INFO:teuthology.orchestra.run.smithi078.stdout: "crash" 2024-09-21T12:38:32.849 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-21T12:38:32.850 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": "18/23 daemons upgraded", 2024-09-21T12:38:32.850 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "Currently upgrading grafana daemons", 2024-09-21T12:38:32.850 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-21T12:38:32.850 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:38:33.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:32 smithi148 ceph-mon[49387]: from='client.34540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:38:33.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:32 smithi148 ceph-mon[49387]: from='client.34544 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:38:33.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:32 smithi148 ceph-mon[49387]: pgmap v349: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 86 B/s rd, 347 B/s wr, 0 op/s 2024-09-21T12:38:33.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:32 smithi148 ceph-mon[49387]: from='client.34548 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:38:33.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:32 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/1483905889' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:33.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:32 smithi078 ceph-mon[63866]: from='client.34540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:38:33.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:32 smithi078 ceph-mon[63866]: from='client.34544 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:38:33.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:32 smithi078 ceph-mon[63866]: pgmap v349: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 86 B/s rd, 347 B/s wr, 0 op/s 2024-09-21T12:38:33.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:32 smithi078 ceph-mon[63866]: from='client.34548 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:38:33.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:32 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/1483905889' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:33.449 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-21T12:38:34.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:33 smithi148 ceph-mon[49387]: from='client.34556 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:38:34.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:33 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/890178598' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:38:34.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:33 smithi078 ceph-mon[63866]: from='client.34556 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:38:34.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:33 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/890178598' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:38:35.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:34 smithi148 ceph-mon[49387]: pgmap v350: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 341 B/s wr, 0 op/s 2024-09-21T12:38:35.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:34 smithi078 ceph-mon[63866]: pgmap v350: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 341 B/s wr, 0 op/s 2024-09-21T12:38:37.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:36 smithi148 ceph-mon[49387]: pgmap v351: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:38:37.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:36 smithi078 ceph-mon[63866]: pgmap v351: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:38:39.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:38 smithi078 ceph-mon[63866]: pgmap v352: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:38:39.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:38 smithi148 ceph-mon[49387]: pgmap v352: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:38:40.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:40 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:40.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:40 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:40.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:40 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:38:40.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:40 smithi148 ceph-mon[49387]: pgmap v353: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:38:40.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:40 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:40.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:40 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:40.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:40 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:38:40.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:40 smithi078 ceph-mon[63866]: pgmap v353: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:38:42.680 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:42 smithi078 ceph-mon[63866]: pgmap v354: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:38:43.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:42 smithi148 ceph-mon[49387]: pgmap v354: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:38:45.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:44 smithi148 ceph-mon[49387]: pgmap v355: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T12:38:45.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:44 smithi078 ceph-mon[63866]: pgmap v355: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T12:38:46.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:38:46.055 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:38:47.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:46 smithi148 ceph-mon[49387]: pgmap v356: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:38:47.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:46 smithi078 ceph-mon[63866]: pgmap v356: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:38:49.048 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:48 smithi078 ceph-mon[63866]: pgmap v357: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:38:49.048 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:48 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:49.048 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:48 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:49.048 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:48 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:49.048 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:48 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:49.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:48 smithi148 ceph-mon[49387]: pgmap v357: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:38:49.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:48 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:49.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:48 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:49.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:48 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:49.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:48 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:50.800 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:50 smithi148 ceph-mon[49387]: pgmap v358: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:38:50.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:50 smithi078 ceph-mon[63866]: pgmap v358: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:38:53.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: pgmap v359: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:38:53.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:53.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:53.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:38:53.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:38:53.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:53.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:53.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-21T12:38:53.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:38:53.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.052 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:53.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T12:38:53.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T12:38:53.053 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T12:38:53.054 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T12:38:53.054 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T12:38:53.054 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T12:38:53.054 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T12:38:53.054 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-21T12:38:53.054 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T12:38:53.054 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-21T12:38:53.054 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T12:38:53.054 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-21T12:38:53.054 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T12:38:53.054 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T12:38:53.054 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T12:38:53.054 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T12:38:53.054 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T12:38:53.055 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T12:38:53.055 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-21T12:38:53.055 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T12:38:53.055 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T12:38:53.055 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T12:38:53.055 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T12:38:53.055 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T12:38:53.055 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T12:38:53.055 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T12:38:53.055 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T12:38:53.055 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T12:38:53.055 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T12:38:53.055 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T12:38:53.055 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T12:38:53.055 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:38:53.056 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:53.056 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:53.056 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:38:53.056 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:38:53.056 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:53.056 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:52 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:53.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: pgmap v359: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:38:53.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:53.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:53.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:38:53.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:38:53.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:53.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:53.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-21T12:38:53.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:38:53.057 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.059 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.059 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.059 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.059 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.059 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:38:53.059 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:53.059 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T12:38:53.059 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T12:38:53.059 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T12:38:53.059 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T12:38:53.059 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T12:38:53.059 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T12:38:53.059 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T12:38:53.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-21T12:38:53.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T12:38:53.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-21T12:38:53.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T12:38:53.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-21T12:38:53.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T12:38:53.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T12:38:53.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T12:38:53.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T12:38:53.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T12:38:53.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T12:38:53.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-21T12:38:53.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T12:38:53.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T12:38:53.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T12:38:53.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T12:38:53.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T12:38:53.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T12:38:53.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T12:38:53.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T12:38:53.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T12:38:53.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T12:38:53.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T12:38:53.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T12:38:53.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:38:53.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:53.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:53.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:38:53.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:38:53.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:53.061 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:52 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:54.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:53 smithi148 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T12:38:54.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:53 smithi148 ceph-mon[49387]: Upgrade: Finalizing container_image settings 2024-09-21T12:38:54.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:53 smithi148 ceph-mon[49387]: Upgrade: Complete! 2024-09-21T12:38:54.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:53 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:38:54.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:53 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:38:54.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:53 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:38:54.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:53 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:54.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:53 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:54.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:53 smithi078 ceph-mon[63866]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T12:38:54.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:53 smithi078 ceph-mon[63866]: Upgrade: Finalizing container_image settings 2024-09-21T12:38:54.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:53 smithi078 ceph-mon[63866]: Upgrade: Complete! 2024-09-21T12:38:54.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:53 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:38:54.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:53 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:38:54.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:53 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:38:54.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:53 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:54.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:53 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:55.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:54 smithi148 ceph-mon[49387]: pgmap v360: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T12:38:55.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:54 smithi078 ceph-mon[63866]: pgmap v360: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T12:38:56.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:55 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:56.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:55 smithi148 ceph-mon[49387]: pgmap v361: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:38:56.305 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:55 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:38:56.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:55 smithi078 ceph-mon[63866]: pgmap v361: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:38:58.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:38:58 smithi148 ceph-mon[49387]: pgmap v362: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:38:58.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:38:58 smithi078 ceph-mon[63866]: pgmap v362: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:01.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:00 smithi148 ceph-mon[49387]: pgmap v363: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:01.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:39:01.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:00 smithi078 ceph-mon[63866]: pgmap v363: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:01.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:39:03.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:02 smithi148 ceph-mon[49387]: pgmap v364: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:03.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:02 smithi078 ceph-mon[63866]: pgmap v364: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:04.583 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -e sha1=441784946648ff53dd310293f9dd752afeeea72a -- bash -c 'ceph orch ps' 2024-09-21T12:39:04.882 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:04 smithi078 ceph-mon[63866]: pgmap v365: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T12:39:04.882 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:04 smithi078 ceph-mon[63866]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:39:05.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:04 smithi148 ceph-mon[49387]: pgmap v365: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T12:39:05.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:04 smithi148 ceph-mon[49387]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:39:06.790 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T12:39:06.790 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (11m) 18s ago 20m 20.2M - 0.27.0 11f11916f8cd e6e67ab07dec 2024-09-21T12:39:06.790 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (7m) 18s ago 20m 6895k - 19.3.0-5104-g44178494 b415816db8b4 46a10ea0b111 2024-09-21T12:39:06.790 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi148 smithi148 running (7m) 48s ago 19m 6895k - 19.3.0-5104-g44178494 b415816db8b4 23deb0615621 2024-09-21T12:39:06.790 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (27s) 18s ago 20m 36.7M - 10.4.8 5aad1d7cf004 261136972e0e 2024-09-21T12:39:06.791 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.ytpzwh smithi078 running (91s) 18s ago 14m 29.6M - 19.3.0-5104-g44178494 b415816db8b4 65fd79480937 2024-09-21T12:39:06.791 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi148.uukezr smithi148 running (69s) 48s ago 14m 12.4M - 19.3.0-5104-g44178494 b415816db8b4 3e9d00854170 2024-09-21T12:39:06.791 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.ntwyan smithi078 *:8443,9283,8765 running (10m) 18s ago 21m 533M - 19.3.0-5104-g44178494 b415816db8b4 a61baedebf43 2024-09-21T12:39:06.791 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi148.oyiqmx smithi148 *:8443,9283,8765 running (9m) 48s ago 19m 458M - 19.3.0-5104-g44178494 b415816db8b4 0255535c6fdb 2024-09-21T12:39:06.791 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (9m) 18s ago 21m 68.5M 2048M 19.3.0-5104-g44178494 b415816db8b4 22b3097efbd8 2024-09-21T12:39:06.791 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi148 smithi148 running (8m) 48s ago 19m 49.5M 2048M 19.3.0-5104-g44178494 b415816db8b4 df8b08509136 2024-09-21T12:39:06.791 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.ydbhru smithi078 *:2049 running (56s) 18s ago 13m 67.2M - 5.9 b415816db8b4 4c82527787a5 2024-09-21T12:39:06.791 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi148.einuzt smithi148 *:2049 running (51s) 48s ago 13m 17.7M - 5.9 b415816db8b4 326968a99b8f 2024-09-21T12:39:06.791 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (11m) 18s ago 19m 10.8M - 1.7.0 72c9c2088986 301adb4e9e22 2024-09-21T12:39:06.791 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi148 smithi148 *:9100 running (10m) 48s ago 19m 11.9M - 1.7.0 72c9c2088986 0cceff4bf342 2024-09-21T12:39:06.791 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (7m) 18s ago 18m 411M 1936M 19.3.0-5104-g44178494 b415816db8b4 7374add97534 2024-09-21T12:39:06.791 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (6m) 18s ago 18m 315M 1936M 19.3.0-5104-g44178494 b415816db8b4 14655a8da4e5 2024-09-21T12:39:06.792 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (5m) 18s ago 17m 295M 1936M 19.3.0-5104-g44178494 b415816db8b4 07ceda32f383 2024-09-21T12:39:06.792 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (4m) 18s ago 17m 384M 1936M 19.3.0-5104-g44178494 b415816db8b4 ead87e4306ff 2024-09-21T12:39:06.792 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi148 running (3m) 48s ago 17m 338M 2704M 19.3.0-5104-g44178494 b415816db8b4 10ec7d6b476e 2024-09-21T12:39:06.792 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi148 running (3m) 48s ago 16m 326M 2704M 19.3.0-5104-g44178494 b415816db8b4 06ac02700ea1 2024-09-21T12:39:06.792 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi148 running (2m) 48s ago 16m 280M 2704M 19.3.0-5104-g44178494 b415816db8b4 17613684a653 2024-09-21T12:39:06.792 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi148 running (118s) 48s ago 15m 299M 2704M 19.3.0-5104-g44178494 b415816db8b4 2e6a6f842c47 2024-09-21T12:39:06.792 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (9m) 18s ago 19m 56.0M - 2.51.0 1d3b7f56885b face2d4cdd1b 2024-09-21T12:39:07.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:06 smithi148 ceph-mon[49387]: pgmap v366: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:07.055 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:06 smithi078 ceph-mon[63866]: pgmap v366: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:07.532 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -e sha1=441784946648ff53dd310293f9dd752afeeea72a -- bash -c 'ceph versions' 2024-09-21T12:39:08.022 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:07 smithi078 ceph-mon[63866]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:39:08.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:07 smithi148 ceph-mon[49387]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:39:08.951 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:08 smithi148 ceph-mon[49387]: pgmap v367: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:08.959 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:08 smithi078 ceph-mon[63866]: pgmap v367: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:09.802 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:39:09.802 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-21T12:39:09.802 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:39:09.802 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:39:09.803 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-21T12:39:09.803 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:39:09.803 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:39:09.803 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-21T12:39:09.803 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 8 2024-09-21T12:39:09.803 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:39:09.803 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-21T12:39:09.803 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:39:09.803 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:39:09.803 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-21T12:39:09.803 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 14 2024-09-21T12:39:09.803 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T12:39:09.803 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:39:10.589 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -e sha1=441784946648ff53dd310293f9dd752afeeea72a -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-09-21T12:39:10.888 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:10 smithi078 ceph-mon[63866]: pgmap v368: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:10.889 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:10 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/2913945892' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:39:11.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:10 smithi148 ceph-mon[49387]: pgmap v368: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:11.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:10 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/2913945892' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:39:12.427 INFO:teuthology.orchestra.run.smithi078.stdout:wait for servicemap items w/ changing names to refresh 2024-09-21T12:39:13.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:12 smithi148 ceph-mon[49387]: pgmap v369: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:13.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:12 smithi078 ceph-mon[63866]: pgmap v369: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:13.192 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -e sha1=441784946648ff53dd310293f9dd752afeeea72a -- bash -c 'sleep 60' 2024-09-21T12:39:14.923 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:14 smithi078 ceph-mon[63866]: pgmap v370: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T12:39:15.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:14 smithi148 ceph-mon[49387]: pgmap v370: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T12:39:16.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:15 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:39:16.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:15 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:39:17.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:16 smithi148 ceph-mon[49387]: pgmap v371: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:17.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:16 smithi078 ceph-mon[63866]: pgmap v371: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:19.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:18 smithi148 ceph-mon[49387]: pgmap v372: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:19.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:18 smithi078 ceph-mon[63866]: pgmap v372: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:21.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:20 smithi148 ceph-mon[49387]: pgmap v373: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:21.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:20 smithi078 ceph-mon[63866]: pgmap v373: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:23.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:22 smithi148 ceph-mon[49387]: pgmap v374: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:23.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:22 smithi078 ceph-mon[63866]: pgmap v374: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:25.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:24 smithi148 ceph-mon[49387]: pgmap v375: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T12:39:25.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:24 smithi078 ceph-mon[63866]: pgmap v375: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T12:39:27.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:26 smithi148 ceph-mon[49387]: pgmap v376: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:27.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:26 smithi078 ceph-mon[63866]: pgmap v376: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:28.301 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:28 smithi148 ceph-mon[49387]: pgmap v377: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:28.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:28 smithi078 ceph-mon[63866]: pgmap v377: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:31.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:30 smithi148 ceph-mon[49387]: pgmap v378: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:31.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:39:31.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:30 smithi078 ceph-mon[63866]: pgmap v378: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:31.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:39:33.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:32 smithi148 ceph-mon[49387]: pgmap v379: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:33.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:32 smithi078 ceph-mon[63866]: pgmap v379: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:35.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:34 smithi148 ceph-mon[49387]: pgmap v380: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T12:39:35.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:34 smithi078 ceph-mon[63866]: pgmap v380: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T12:39:37.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:36 smithi148 ceph-mon[49387]: pgmap v381: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:37.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:36 smithi078 ceph-mon[63866]: pgmap v381: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:39.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:38 smithi148 ceph-mon[49387]: pgmap v382: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:39.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:38 smithi078 ceph-mon[63866]: pgmap v382: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:41.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:40 smithi148 ceph-mon[49387]: pgmap v383: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:41.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:40 smithi078 ceph-mon[63866]: pgmap v383: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:43.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:42 smithi148 ceph-mon[49387]: pgmap v384: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:43.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:42 smithi078 ceph-mon[63866]: pgmap v384: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:45.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:44 smithi148 ceph-mon[49387]: pgmap v385: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T12:39:45.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:44 smithi078 ceph-mon[63866]: pgmap v385: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T12:39:46.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:39:46.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:39:47.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:46 smithi148 ceph-mon[49387]: pgmap v386: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:47.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:46 smithi078 ceph-mon[63866]: pgmap v386: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:49.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:48 smithi148 ceph-mon[49387]: pgmap v387: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:49.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:48 smithi078 ceph-mon[63866]: pgmap v387: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:51.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:50 smithi148 ceph-mon[49387]: pgmap v388: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:51.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:50 smithi078 ceph-mon[63866]: pgmap v388: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:53.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:52 smithi148 ceph-mon[49387]: pgmap v389: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:53.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:52 smithi078 ceph-mon[63866]: pgmap v389: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:54.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:53 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:39:54.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:53 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:39:54.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:53 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:39:54.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:53 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T12:39:54.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:53 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:39:54.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:53 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:39:55.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:54 smithi148 ceph-mon[49387]: pgmap v390: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T12:39:55.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:54 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:39:55.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:54 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:39:55.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:54 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:39:55.051 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:54 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:39:55.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:54 smithi078 ceph-mon[63866]: pgmap v390: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T12:39:55.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:54 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T12:39:55.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:54 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T12:39:55.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:54 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:39:55.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:54 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' 2024-09-21T12:39:57.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:56 smithi148 ceph-mon[49387]: pgmap v391: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:57.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:56 smithi078 ceph-mon[63866]: pgmap v391: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:59.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:39:58 smithi148 ceph-mon[49387]: pgmap v392: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:39:59.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:39:58 smithi078 ceph-mon[63866]: pgmap v392: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:01.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:00 smithi148 ceph-mon[49387]: pgmap v393: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:01.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:00 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:40:01.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:00 smithi148 ceph-mon[49387]: overall HEALTH_OK 2024-09-21T12:40:01.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:00 smithi078 ceph-mon[63866]: pgmap v393: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:01.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:00 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:40:01.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:00 smithi078 ceph-mon[63866]: overall HEALTH_OK 2024-09-21T12:40:03.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:02 smithi148 ceph-mon[49387]: pgmap v394: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:03.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:02 smithi078 ceph-mon[63866]: pgmap v394: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:05.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:04 smithi148 ceph-mon[49387]: pgmap v395: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T12:40:05.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:04 smithi078 ceph-mon[63866]: pgmap v395: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T12:40:07.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:06 smithi148 ceph-mon[49387]: pgmap v396: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:07.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:06 smithi078 ceph-mon[63866]: pgmap v396: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:09.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:08 smithi148 ceph-mon[49387]: pgmap v397: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:09.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:08 smithi078 ceph-mon[63866]: pgmap v397: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:11.043 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:10 smithi148 ceph-mon[49387]: pgmap v398: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:11.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:10 smithi078 ceph-mon[63866]: pgmap v398: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:13.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:12 smithi148 ceph-mon[49387]: pgmap v399: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:13.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:12 smithi078 ceph-mon[63866]: pgmap v399: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:15.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:14 smithi148 ceph-mon[49387]: pgmap v400: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T12:40:15.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:14 smithi078 ceph-mon[63866]: pgmap v400: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T12:40:15.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:15 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:40:15.923 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -e sha1=441784946648ff53dd310293f9dd752afeeea72a -- bash -c 'ceph orch ps' 2024-09-21T12:40:16.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:15 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:40:16.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:16 smithi078 ceph-mon[63866]: pgmap v401: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:17.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:16 smithi148 ceph-mon[49387]: pgmap v401: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:18.176 INFO:teuthology.orchestra.run.smithi078.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T12:40:18.176 INFO:teuthology.orchestra.run.smithi078.stdout:alertmanager.smithi078 smithi078 *:9093,9094 running (12m) 90s ago 21m 20.2M - 0.27.0 11f11916f8cd e6e67ab07dec 2024-09-21T12:40:18.176 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi078 smithi078 running (9m) 90s ago 21m 6895k - 19.3.0-5104-g44178494 b415816db8b4 46a10ea0b111 2024-09-21T12:40:18.176 INFO:teuthology.orchestra.run.smithi078.stdout:crash.smithi148 smithi148 running (9m) 119s ago 20m 6895k - 19.3.0-5104-g44178494 b415816db8b4 23deb0615621 2024-09-21T12:40:18.176 INFO:teuthology.orchestra.run.smithi078.stdout:grafana.smithi078 smithi078 *:3000 running (99s) 90s ago 21m 36.7M - 10.4.8 5aad1d7cf004 261136972e0e 2024-09-21T12:40:18.176 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi078.ytpzwh smithi078 running (2m) 90s ago 15m 29.6M - 19.3.0-5104-g44178494 b415816db8b4 65fd79480937 2024-09-21T12:40:18.176 INFO:teuthology.orchestra.run.smithi078.stdout:mds.foofs.smithi148.uukezr smithi148 running (2m) 119s ago 15m 12.4M - 19.3.0-5104-g44178494 b415816db8b4 3e9d00854170 2024-09-21T12:40:18.176 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi078.ntwyan smithi078 *:8443,9283,8765 running (11m) 90s ago 22m 533M - 19.3.0-5104-g44178494 b415816db8b4 a61baedebf43 2024-09-21T12:40:18.177 INFO:teuthology.orchestra.run.smithi078.stdout:mgr.smithi148.oyiqmx smithi148 *:8443,9283,8765 running (10m) 119s ago 20m 458M - 19.3.0-5104-g44178494 b415816db8b4 0255535c6fdb 2024-09-21T12:40:18.177 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi078 smithi078 running (10m) 90s ago 22m 68.5M 2048M 19.3.0-5104-g44178494 b415816db8b4 22b3097efbd8 2024-09-21T12:40:18.177 INFO:teuthology.orchestra.run.smithi078.stdout:mon.smithi148 smithi148 running (10m) 119s ago 20m 49.5M 2048M 19.3.0-5104-g44178494 b415816db8b4 df8b08509136 2024-09-21T12:40:18.177 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.0.0.smithi078.ydbhru smithi078 *:2049 running (2m) 90s ago 15m 67.2M - 5.9 b415816db8b4 4c82527787a5 2024-09-21T12:40:18.177 INFO:teuthology.orchestra.run.smithi078.stdout:nfs.foo.1.0.smithi148.einuzt smithi148 *:2049 running (2m) 119s ago 15m 17.7M - 5.9 b415816db8b4 326968a99b8f 2024-09-21T12:40:18.177 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi078 smithi078 *:9100 running (12m) 90s ago 21m 10.8M - 1.7.0 72c9c2088986 301adb4e9e22 2024-09-21T12:40:18.177 INFO:teuthology.orchestra.run.smithi078.stdout:node-exporter.smithi148 smithi148 *:9100 running (11m) 119s ago 20m 11.9M - 1.7.0 72c9c2088986 0cceff4bf342 2024-09-21T12:40:18.177 INFO:teuthology.orchestra.run.smithi078.stdout:osd.0 smithi078 running (8m) 90s ago 19m 411M 1936M 19.3.0-5104-g44178494 b415816db8b4 7374add97534 2024-09-21T12:40:18.177 INFO:teuthology.orchestra.run.smithi078.stdout:osd.1 smithi078 running (7m) 90s ago 19m 315M 1936M 19.3.0-5104-g44178494 b415816db8b4 14655a8da4e5 2024-09-21T12:40:18.177 INFO:teuthology.orchestra.run.smithi078.stdout:osd.2 smithi078 running (6m) 90s ago 19m 295M 1936M 19.3.0-5104-g44178494 b415816db8b4 07ceda32f383 2024-09-21T12:40:18.177 INFO:teuthology.orchestra.run.smithi078.stdout:osd.3 smithi078 running (5m) 90s ago 18m 384M 1936M 19.3.0-5104-g44178494 b415816db8b4 ead87e4306ff 2024-09-21T12:40:18.177 INFO:teuthology.orchestra.run.smithi078.stdout:osd.4 smithi148 running (5m) 119s ago 18m 338M 2704M 19.3.0-5104-g44178494 b415816db8b4 10ec7d6b476e 2024-09-21T12:40:18.178 INFO:teuthology.orchestra.run.smithi078.stdout:osd.5 smithi148 running (4m) 119s ago 17m 326M 2704M 19.3.0-5104-g44178494 b415816db8b4 06ac02700ea1 2024-09-21T12:40:18.178 INFO:teuthology.orchestra.run.smithi078.stdout:osd.6 smithi148 running (3m) 119s ago 17m 280M 2704M 19.3.0-5104-g44178494 b415816db8b4 17613684a653 2024-09-21T12:40:18.178 INFO:teuthology.orchestra.run.smithi078.stdout:osd.7 smithi148 running (3m) 119s ago 16m 299M 2704M 19.3.0-5104-g44178494 b415816db8b4 2e6a6f842c47 2024-09-21T12:40:18.178 INFO:teuthology.orchestra.run.smithi078.stdout:prometheus.smithi078 smithi078 *:9095 running (10m) 90s ago 20m 56.0M - 2.51.0 1d3b7f56885b face2d4cdd1b 2024-09-21T12:40:18.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:18 smithi078 ceph-mon[63866]: pgmap v402: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:18.883 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -e sha1=441784946648ff53dd310293f9dd752afeeea72a -- bash -c 'ceph orch upgrade status' 2024-09-21T12:40:19.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:18 smithi148 ceph-mon[49387]: pgmap v402: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:19.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:19 smithi078 ceph-mon[63866]: from='client.34576 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:40:20.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:19 smithi148 ceph-mon[49387]: from='client.34576 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:40:20.874 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:40:20.875 INFO:teuthology.orchestra.run.smithi078.stdout: "target_image": null, 2024-09-21T12:40:20.875 INFO:teuthology.orchestra.run.smithi078.stdout: "in_progress": false, 2024-09-21T12:40:20.875 INFO:teuthology.orchestra.run.smithi078.stdout: "which": "", 2024-09-21T12:40:20.875 INFO:teuthology.orchestra.run.smithi078.stdout: "services_complete": [], 2024-09-21T12:40:20.875 INFO:teuthology.orchestra.run.smithi078.stdout: "progress": null, 2024-09-21T12:40:20.875 INFO:teuthology.orchestra.run.smithi078.stdout: "message": "", 2024-09-21T12:40:20.875 INFO:teuthology.orchestra.run.smithi078.stdout: "is_paused": false 2024-09-21T12:40:20.875 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:40:21.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:20 smithi148 ceph-mon[49387]: pgmap v403: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:21.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:20 smithi078 ceph-mon[63866]: pgmap v403: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:21.539 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -e sha1=441784946648ff53dd310293f9dd752afeeea72a -- bash -c 'ceph health detail' 2024-09-21T12:40:21.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:21 smithi078 ceph-mon[63866]: from='client.34580 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:40:22.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:21 smithi148 ceph-mon[49387]: from='client.34580 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:40:23.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:22 smithi148 ceph-mon[49387]: pgmap v404: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:23.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:22 smithi078 ceph-mon[63866]: pgmap v404: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:23.843 INFO:teuthology.orchestra.run.smithi078.stdout:HEALTH_OK 2024-09-21T12:40:24.562 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -e sha1=441784946648ff53dd310293f9dd752afeeea72a -- bash -c 'ceph versions' 2024-09-21T12:40:24.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:24 smithi078 ceph-mon[63866]: pgmap v405: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T12:40:24.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:24 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/413772279' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:40:25.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:24 smithi148 ceph-mon[49387]: pgmap v405: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T12:40:25.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:24 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/413772279' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-21T12:40:26.684 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-09-21T12:40:26.684 INFO:teuthology.orchestra.run.smithi078.stdout: "mon": { 2024-09-21T12:40:26.684 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:40:26.685 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:40:26.685 INFO:teuthology.orchestra.run.smithi078.stdout: "mgr": { 2024-09-21T12:40:26.685 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:40:26.685 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:40:26.685 INFO:teuthology.orchestra.run.smithi078.stdout: "osd": { 2024-09-21T12:40:26.685 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 8 2024-09-21T12:40:26.685 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:40:26.685 INFO:teuthology.orchestra.run.smithi078.stdout: "mds": { 2024-09-21T12:40:26.685 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 2 2024-09-21T12:40:26.685 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:40:26.685 INFO:teuthology.orchestra.run.smithi078.stdout: "overall": { 2024-09-21T12:40:26.685 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)": 14 2024-09-21T12:40:26.686 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T12:40:26.686 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-09-21T12:40:27.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:26 smithi148 ceph-mon[49387]: pgmap v406: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:27.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:26 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/2297123716' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:40:27.053 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:26 smithi078 ceph-mon[63866]: pgmap v406: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:27.053 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:26 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/2297123716' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:40:27.387 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -e sha1=441784946648ff53dd310293f9dd752afeeea72a -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-09-21T12:40:28.997 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:28 smithi078 ceph-mon[63866]: pgmap v407: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:29.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:28 smithi148 ceph-mon[49387]: pgmap v407: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:29.575 INFO:teuthology.orchestra.run.smithi078.stdout:true 2024-09-21T12:40:29.876 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:29 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/1702779946' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:40:30.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:29 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/1702779946' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:40:30.202 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -e sha1=441784946648ff53dd310293f9dd752afeeea72a -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-09-21T12:40:31.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:30 smithi148 ceph-mon[49387]: pgmap v408: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:31.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:30 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:40:31.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:30 smithi078 ceph-mon[63866]: pgmap v408: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:31.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:30 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:40:32.346 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph version 19.3.0-5104-g44178494 (441784946648ff53dd310293f9dd752afeeea72a) squid (dev)" 2024-09-21T12:40:33.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:32 smithi148 ceph-mon[49387]: pgmap v409: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:33.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:32 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/768371449' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:40:33.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:32 smithi078 ceph-mon[63866]: pgmap v409: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:33.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:32 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/768371449' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T12:40:33.136 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-09-21T12:40:33.136 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-09-21T12:40:33.136 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- ceph orch ls -f json 2024-09-21T12:40:35.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:34 smithi148 ceph-mon[49387]: pgmap v410: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T12:40:35.053 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-09-21T12:40:35.053 INFO:teuthology.orchestra.run.smithi078.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-21T12:18:11.788478Z", "last_refresh": "2024-09-21T12:38:47.911571Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-21T12:18:05.156920Z", "last_refresh": "2024-09-21T12:38:18.735300Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-09-21T12:18:08.632362Z", "last_refresh": "2024-09-21T12:38:47.911709Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-21T12:24:45.253345Z", "last_refresh": "2024-09-21T12:38:18.735911Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-21T12:18:03.699549Z", "last_refresh": "2024-09-21T12:38:18.735410Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi078:172.21.15.78=smithi078", "smithi148:172.21.15.148=smithi148"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-21T12:18:55.548938Z", "last_refresh": "2024-09-21T12:38:18.735479Z", "running": 2, "size": 2}}, {"events": ["2024-09-21T12:39:53.836960Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-09-21T12:25:05.681846Z", "last_refresh": "2024-09-21T12:38:18.735980Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-21T12:18:10.264330Z", "last_refresh": "2024-09-21T12:38:18.735545Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5", "last_refresh": "2024-09-21T12:38:18.735629Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-21T12:18:06.747420Z", "last_refresh": "2024-09-21T12:38:47.911846Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-21T12:40:35.054 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:34 smithi078 ceph-mon[63866]: pgmap v410: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-21T12:40:35.688 INFO:tasks.cephadm:nfs.foo has 2/2 2024-09-21T12:40:35.689 DEBUG:teuthology.parallel:result is None 2024-09-21T12:40:35.689 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-21T12:40:35.697 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-21T12:40:35.697 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-09-21T12:40:35.723 INFO:teuthology.orchestra.run.smithi078.stderr:+ umount /mnt/foo 2024-09-21T12:40:35.746 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-21T12:40:35.755 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-21T12:40:35.755 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-09-21T12:40:37.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:36 smithi148 ceph-mon[49387]: from='client.34600 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T12:40:37.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:36 smithi148 ceph-mon[49387]: pgmap v411: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:37.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:36 smithi078 ceph-mon[63866]: from='client.34600 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T12:40:37.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:36 smithi078 ceph-mon[63866]: pgmap v411: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:37.923 INFO:teuthology.orchestra.run.smithi078.stdout: "foo" 2024-09-21T12:40:38.562 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- bash -c 'ceph nfs export ls foo --detailed' 2024-09-21T12:40:38.806 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:38 smithi078 ceph-mon[63866]: pgmap v412: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:39.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:38 smithi148 ceph-mon[49387]: pgmap v412: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:39.903 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:39 smithi078 ceph-mon[63866]: from='client.34604 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:40:40.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:39 smithi148 ceph-mon[49387]: from='client.34604 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:40:40.759 INFO:teuthology.orchestra.run.smithi078.stdout:[ 2024-09-21T12:40:40.759 INFO:teuthology.orchestra.run.smithi078.stdout: { 2024-09-21T12:40:40.760 INFO:teuthology.orchestra.run.smithi078.stdout: "access_type": "RW", 2024-09-21T12:40:40.760 INFO:teuthology.orchestra.run.smithi078.stdout: "clients": [], 2024-09-21T12:40:40.760 INFO:teuthology.orchestra.run.smithi078.stdout: "cluster_id": "foo", 2024-09-21T12:40:40.760 INFO:teuthology.orchestra.run.smithi078.stdout: "export_id": 1, 2024-09-21T12:40:40.760 INFO:teuthology.orchestra.run.smithi078.stdout: "fsal": { 2024-09-21T12:40:40.760 INFO:teuthology.orchestra.run.smithi078.stdout: "fs_name": "foofs", 2024-09-21T12:40:40.760 INFO:teuthology.orchestra.run.smithi078.stdout: "name": "CEPH", 2024-09-21T12:40:40.760 INFO:teuthology.orchestra.run.smithi078.stdout: "user_id": "nfs.foo.1" 2024-09-21T12:40:40.760 INFO:teuthology.orchestra.run.smithi078.stdout: }, 2024-09-21T12:40:40.760 INFO:teuthology.orchestra.run.smithi078.stdout: "path": "/", 2024-09-21T12:40:40.760 INFO:teuthology.orchestra.run.smithi078.stdout: "protocols": [ 2024-09-21T12:40:40.761 INFO:teuthology.orchestra.run.smithi078.stdout: 4 2024-09-21T12:40:40.761 INFO:teuthology.orchestra.run.smithi078.stdout: ], 2024-09-21T12:40:40.761 INFO:teuthology.orchestra.run.smithi078.stdout: "pseudo": "/fake", 2024-09-21T12:40:40.761 INFO:teuthology.orchestra.run.smithi078.stdout: "security_label": true, 2024-09-21T12:40:40.761 INFO:teuthology.orchestra.run.smithi078.stdout: "squash": "none", 2024-09-21T12:40:40.761 INFO:teuthology.orchestra.run.smithi078.stdout: "transports": [ 2024-09-21T12:40:40.761 INFO:teuthology.orchestra.run.smithi078.stdout: "TCP" 2024-09-21T12:40:40.761 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-09-21T12:40:40.761 INFO:teuthology.orchestra.run.smithi078.stdout: } 2024-09-21T12:40:40.761 INFO:teuthology.orchestra.run.smithi078.stdout:] 2024-09-21T12:40:41.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:40 smithi148 ceph-mon[49387]: pgmap v413: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:41.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:40 smithi078 ceph-mon[63866]: pgmap v413: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:41.468 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- bash -c 'rados -p .nfs --all ls -' 2024-09-21T12:40:42.047 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:41 smithi078 ceph-mon[63866]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:40:42.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:41 smithi148 ceph-mon[49387]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T12:40:43.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:42 smithi148 ceph-mon[49387]: pgmap v414: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:43.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:42 smithi148 ceph-mon[49387]: mgrmap e44: smithi078.ntwyan(active, since 10m), standbys: smithi148.oyiqmx 2024-09-21T12:40:43.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:42 smithi078 ceph-mon[63866]: pgmap v414: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-21T12:40:43.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:42 smithi078 ceph-mon[63866]: mgrmap e44: smithi078.ntwyan(active, since 10m), standbys: smithi148.oyiqmx 2024-09-21T12:40:43.295 INFO:teuthology.orchestra.run.smithi078.stdout:foo rec-0000000000000004:nfs.foo.1 2024-09-21T12:40:43.295 INFO:teuthology.orchestra.run.smithi078.stdout:foo grace 2024-09-21T12:40:43.295 INFO:teuthology.orchestra.run.smithi078.stdout:foo rec-0000000000000004:nfs.foo.0 2024-09-21T12:40:43.296 INFO:teuthology.orchestra.run.smithi078.stdout:foo conf-nfs.foo 2024-09-21T12:40:43.296 INFO:teuthology.orchestra.run.smithi078.stdout:foo export-1 2024-09-21T12:40:43.928 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-21T12:40:43.938 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi078.front.sepia.ceph.com 2024-09-21T12:40:43.938 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 6263deca-7813-11ef-baf6-efdc52797490 -- bash -c 'set -ex 2024-09-21T12:40:43.939 DEBUG:teuthology.orchestra.run.smithi078:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-09-21T12:40:43.939 DEBUG:teuthology.orchestra.run.smithi078:> ' 2024-09-21T12:40:45.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:44 smithi148 ceph-mon[49387]: pgmap v415: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-21T12:40:45.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:44 smithi078 ceph-mon[63866]: pgmap v415: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-21T12:40:45.770 INFO:teuthology.orchestra.run.smithi078.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-09-21T12:40:46.002 INFO:teuthology.orchestra.run.smithi078.stderr:+ [[ 7 -gt 2 ]] 2024-09-21T12:40:46.056 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:45 smithi078 ceph-mon[63866]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:40:46.300 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:45 smithi148 ceph-mon[49387]: from='mgr.34100 172.21.15.78:0/2425034545' entity='mgr.smithi078.ntwyan' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T12:40:46.581 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-21T12:40:46.599 INFO:tasks.cephadm:Teardown begin 2024-09-21T12:40:46.599 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T12:40:46.626 DEBUG:teuthology.orchestra.run.smithi148:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T12:40:46.660 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-21T12:40:46.660 DEBUG:teuthology.orchestra.run.smithi078:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-21T12:40:46.682 DEBUG:teuthology.orchestra.run.smithi148:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-21T12:40:46.717 INFO:tasks.cephadm:Stopping all daemons... 2024-09-21T12:40:46.717 INFO:tasks.cephadm.mon.smithi078:Stopping mon.smithi078... 2024-09-21T12:40:46.718 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-6263deca-7813-11ef-baf6-efdc52797490@mon.smithi078 2024-09-21T12:40:46.758 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:46 smithi078 systemd[1]: Stopping Ceph mon.smithi078 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:40:47.011 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:46 smithi078 ceph-mon[63866]: pgmap v416: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-21T12:40:47.012 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:46 smithi078 ceph-mon[63866]: from='client.? 172.21.15.78:0/23767280' entity='client.admin' cmd={"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"} : dispatch 2024-09-21T12:40:47.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:46 smithi148 ceph-mon[49387]: pgmap v416: 97 pgs: 97 active+clean; 229 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-21T12:40:47.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:46 smithi148 ceph-mon[49387]: from='client.? 172.21.15.78:0/23767280' entity='client.admin' cmd={"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"} : dispatch 2024-09-21T12:40:47.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:47 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi078[63862]: 2024-09-21T12:40:47.009+0000 7f79f7c3c640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi078 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T12:40:47.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:47 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi078[63862]: 2024-09-21T12:40:47.009+0000 7f79f7c3c640 -1 mon.smithi078@0(leader) e3 *** Got Signal Terminated *** 2024-09-21T12:40:47.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:47 smithi078 podman[93163]: 2024-09-21 12:40:47.041094657 +0000 UTC m=+0.138598332 container died 22b3097efbd8c02c4212806c49ca77f908da21e3a296e763947f8da487f3b6db (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi078, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T12:40:47.654 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:47 smithi078 podman[93163]: 2024-09-21 12:40:47.460086223 +0000 UTC m=+0.557589890 container cleanup 22b3097efbd8c02c4212806c49ca77f908da21e3a296e763947f8da487f3b6db (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi078, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-21T12:40:47.654 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:47 smithi078 bash[93163]: ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi078 2024-09-21T12:40:47.896 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-6263deca-7813-11ef-baf6-efdc52797490@mon.smithi078.service' 2024-09-21T12:40:47.959 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:47 smithi078 podman[93175]: 2024-09-21 12:40:47.653240501 +0000 UTC m=+0.614896771 container remove 22b3097efbd8c02c4212806c49ca77f908da21e3a296e763947f8da487f3b6db (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi078, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-21-0842-4417849) 2024-09-21T12:40:47.959 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:47 smithi078 systemd[1]: ceph-6263deca-7813-11ef-baf6-efdc52797490@mon.smithi078.service: Deactivated successfully. 2024-09-21T12:40:47.959 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:47 smithi078 systemd[1]: Stopped Ceph mon.smithi078 for 6263deca-7813-11ef-baf6-efdc52797490. 2024-09-21T12:40:47.959 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:Sep 21 12:40:47 smithi078 systemd[1]: ceph-6263deca-7813-11ef-baf6-efdc52797490@mon.smithi078.service: Consumed 13.568s CPU time. 2024-09-21T12:40:47.989 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T12:40:47.990 INFO:tasks.cephadm.mon.smithi078:Stopped mon.smithi078 2024-09-21T12:40:47.990 INFO:tasks.cephadm.mon.smithi148:Stopping mon.smithi148... 2024-09-21T12:40:47.990 DEBUG:teuthology.orchestra.run.smithi148:> sudo systemctl stop ceph-6263deca-7813-11ef-baf6-efdc52797490@mon.smithi148 2024-09-21T12:40:48.293 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:48 smithi148 systemd[1]: Stopping Ceph mon.smithi148 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:40:48.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:48 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi148[49383]: 2024-09-21T12:40:48.290+0000 7f05ef0e5640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi148 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T12:40:48.550 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:48 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi148[49383]: 2024-09-21T12:40:48.290+0000 7f05ef0e5640 -1 mon.smithi148@1(peon) e3 *** Got Signal Terminated *** 2024-09-21T12:40:48.551 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:48 smithi148 podman[68955]: 2024-09-21 12:40:48.475623647 +0000 UTC m=+0.290069651 container died df8b0850913677c8d686548f8c3522af5680c9db06e6cd852dd2ed1ec5d0a17a (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi148, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0842-4417849, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-21T12:40:49.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:48 smithi148 podman[68955]: 2024-09-21 12:40:48.975290977 +0000 UTC m=+0.789737012 container cleanup df8b0850913677c8d686548f8c3522af5680c9db06e6cd852dd2ed1ec5d0a17a (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi148, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0842-4417849) 2024-09-21T12:40:49.050 INFO:journalctl@ceph.mon.smithi148.smithi148.stdout:Sep 21 12:40:48 smithi148 bash[68955]: ceph-6263deca-7813-11ef-baf6-efdc52797490-mon-smithi148 2024-09-21T12:40:49.416 DEBUG:teuthology.orchestra.run.smithi148:> sudo pkill -f 'journalctl -f -n 0 -u ceph-6263deca-7813-11ef-baf6-efdc52797490@mon.smithi148.service' 2024-09-21T12:40:49.466 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T12:40:49.466 INFO:tasks.cephadm.mon.smithi148:Stopped mon.smithi148 2024-09-21T12:40:49.466 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-21T12:40:49.466 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.0 2024-09-21T12:40:49.749 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:40:49 smithi078 systemd[1]: Stopping Ceph osd.0 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:40:50.056 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:40:49 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0[68705]: 2024-09-21T12:40:49.747+0000 7f9f3cbaf640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T12:40:50.056 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:40:49 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0[68705]: 2024-09-21T12:40:49.747+0000 7f9f3cbaf640 -1 osd.0 162 *** Got signal Terminated *** 2024-09-21T12:40:50.056 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:40:49 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0[68705]: 2024-09-21T12:40:49.747+0000 7f9f3cbaf640 -1 osd.0 162 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T12:40:55.500 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:40:55 smithi078 podman[93296]: 2024-09-21 12:40:55.236618578 +0000 UTC m=+5.581718001 container died 7374add9753462c8ad5b0ccb253024e1501e9e80b11efd84c83f38e19b61d493 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True) 2024-09-21T12:40:55.806 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:40:55 smithi078 podman[93296]: 2024-09-21 12:40:55.581217597 +0000 UTC m=+5.926317020 container cleanup 7374add9753462c8ad5b0ccb253024e1501e9e80b11efd84c83f38e19b61d493 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-21T12:40:55.806 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:40:55 smithi078 bash[93296]: ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0 2024-09-21T12:40:56.234 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:40:55 smithi078 podman[93327]: 2024-09-21 12:40:55.890863149 +0000 UTC m=+0.655779731 container remove 7374add9753462c8ad5b0ccb253024e1501e9e80b11efd84c83f38e19b61d493 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-21-0842-4417849, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-21T12:40:56.556 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:40:56 smithi078 podman[93415]: 2024-09-21 12:40:56.233449618 +0000 UTC m=+0.100850184 container create 10a9f57009dfba1b74436bf16331b83e98aee066c571e1082ea3ddb6d7c66460 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0842-4417849, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T12:40:56.556 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:40:56 smithi078 podman[93415]: 2024-09-21 12:40:56.148684623 +0000 UTC m=+0.016085200 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:40:56.556 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:40:56 smithi078 podman[93415]: 2024-09-21 12:40:56.453283537 +0000 UTC m=+0.320684115 container init 10a9f57009dfba1b74436bf16331b83e98aee066c571e1082ea3ddb6d7c66460 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-21-0842-4417849, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T12:40:56.556 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:40:56 smithi078 podman[93415]: 2024-09-21 12:40:56.456976707 +0000 UTC m=+0.324377281 container start 10a9f57009dfba1b74436bf16331b83e98aee066c571e1082ea3ddb6d7c66460 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-21-0842-4417849, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-21T12:40:56.983 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:40:56 smithi078 podman[93415]: 2024-09-21 12:40:56.561068691 +0000 UTC m=+0.428469274 container attach 10a9f57009dfba1b74436bf16331b83e98aee066c571e1082ea3ddb6d7c66460 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0-deactivate, RELEASE=wip-guits-main-2024-09-21-0842-4417849, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=) 2024-09-21T12:40:56.983 INFO:journalctl@ceph.osd.0.smithi078.stdout:Sep 21 12:40:56 smithi078 podman[93415]: 2024-09-21 12:40:56.648743428 +0000 UTC m=+0.516144010 container died 10a9f57009dfba1b74436bf16331b83e98aee066c571e1082ea3ddb6d7c66460 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-0-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-21T12:40:57.349 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.0.service' 2024-09-21T12:40:57.393 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T12:40:57.394 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-21T12:40:57.394 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-21T12:40:57.394 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.1 2024-09-21T12:40:57.727 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:40:57 smithi078 systemd[1]: Stopping Ceph osd.1 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:40:58.056 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:40:57 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1[71271]: 2024-09-21T12:40:57.725+0000 7fd0673dc640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T12:40:58.056 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:40:57 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1[71271]: 2024-09-21T12:40:57.725+0000 7fd0673dc640 -1 osd.1 162 *** Got signal Terminated *** 2024-09-21T12:40:58.056 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:40:57 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1[71271]: 2024-09-21T12:40:57.725+0000 7fd0673dc640 -1 osd.1 162 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T12:41:03.556 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:41:03 smithi078 podman[93518]: 2024-09-21 12:41:03.157302556 +0000 UTC m=+5.534016889 container died 14655a8da4e56dfa60e289d895691a7ad0a0918e65acfcd8a22bfeadc4e1240e (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T12:41:04.037 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:41:03 smithi078 podman[93518]: 2024-09-21 12:41:03.751236867 +0000 UTC m=+6.127951184 container cleanup 14655a8da4e56dfa60e289d895691a7ad0a0918e65acfcd8a22bfeadc4e1240e (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T12:41:04.037 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:41:03 smithi078 bash[93518]: ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1 2024-09-21T12:41:04.306 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:41:04 smithi078 podman[93535]: 2024-09-21 12:41:04.035960693 +0000 UTC m=+0.873634655 container remove 14655a8da4e56dfa60e289d895691a7ad0a0918e65acfcd8a22bfeadc4e1240e (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T12:41:04.606 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:41:04 smithi078 podman[93623]: 2024-09-21 12:41:04.273537189 +0000 UTC m=+0.015862528 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:41:04.607 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:41:04 smithi078 podman[93623]: 2024-09-21 12:41:04.38703401 +0000 UTC m=+0.129359349 container create ad04557aaa61123d30db7c8b8895805ba58605d4a1cbda1287c81dccc72adbd1 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1-deactivate, RELEASE=wip-guits-main-2024-09-21-0842-4417849, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True) 2024-09-21T12:41:04.607 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:41:04 smithi078 podman[93623]: 2024-09-21 12:41:04.605962666 +0000 UTC m=+0.348288018 container init ad04557aaa61123d30db7c8b8895805ba58605d4a1cbda1287c81dccc72adbd1 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-guits-main-2024-09-21-0842-4417849, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-21T12:41:05.056 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:41:04 smithi078 podman[93623]: 2024-09-21 12:41:04.609574671 +0000 UTC m=+0.351900013 container start ad04557aaa61123d30db7c8b8895805ba58605d4a1cbda1287c81dccc72adbd1 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0842-4417849, CEPH_POINT_RELEASE=) 2024-09-21T12:41:05.056 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:41:04 smithi078 podman[93623]: 2024-09-21 12:41:04.6892537 +0000 UTC m=+0.431579049 container attach ad04557aaa61123d30db7c8b8895805ba58605d4a1cbda1287c81dccc72adbd1 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-21T12:41:05.056 INFO:journalctl@ceph.osd.1.smithi078.stdout:Sep 21 12:41:04 smithi078 podman[93623]: 2024-09-21 12:41:04.79854149 +0000 UTC m=+0.540866832 container died ad04557aaa61123d30db7c8b8895805ba58605d4a1cbda1287c81dccc72adbd1 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-1-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.license=GPLv2) 2024-09-21T12:41:05.445 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.1.service' 2024-09-21T12:41:05.487 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T12:41:05.487 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-21T12:41:05.487 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-21T12:41:05.487 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.2 2024-09-21T12:41:05.806 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:41:05 smithi078 systemd[1]: Stopping Ceph osd.2 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:41:06.306 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:41:05 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2[73716]: 2024-09-21T12:41:05.838+0000 7f438f287640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T12:41:06.306 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:41:05 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2[73716]: 2024-09-21T12:41:05.838+0000 7f438f287640 -1 osd.2 162 *** Got signal Terminated *** 2024-09-21T12:41:06.306 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:41:05 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2[73716]: 2024-09-21T12:41:05.838+0000 7f438f287640 -1 osd.2 162 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T12:41:11.698 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:41:11 smithi078 podman[93724]: 2024-09-21 12:41:11.413293314 +0000 UTC m=+5.706041229 container died 07ceda32f383726003b3f2389e763a2ec80e9d0cbbc4edf9f4a361f4f2fc5c2a (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-21T12:41:12.056 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:41:11 smithi078 podman[93724]: 2024-09-21 12:41:11.779203839 +0000 UTC m=+6.071951754 container cleanup 07ceda32f383726003b3f2389e763a2ec80e9d0cbbc4edf9f4a361f4f2fc5c2a (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0842-4417849, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-21T12:41:12.056 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:41:11 smithi078 bash[93724]: ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2 2024-09-21T12:41:12.397 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:41:12 smithi078 podman[93742]: 2024-09-21 12:41:12.063906641 +0000 UTC m=+0.646329532 container remove 07ceda32f383726003b3f2389e763a2ec80e9d0cbbc4edf9f4a361f4f2fc5c2a (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-21T12:41:12.726 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:41:12 smithi078 podman[93845]: 2024-09-21 12:41:12.299548197 +0000 UTC m=+0.015813897 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:41:12.726 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:41:12 smithi078 podman[93845]: 2024-09-21 12:41:12.414660903 +0000 UTC m=+0.130926589 container create c746712e4c789966ffec1d580a66ef27f4fa034da0d9d1cc8aec983ca5b1bf7b (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-21T12:41:12.726 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:41:12 smithi078 podman[93845]: 2024-09-21 12:41:12.642130724 +0000 UTC m=+0.358396411 container init c746712e4c789966ffec1d580a66ef27f4fa034da0d9d1cc8aec983ca5b1bf7b (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2) 2024-09-21T12:41:12.727 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:41:12 smithi078 podman[93845]: 2024-09-21 12:41:12.645792187 +0000 UTC m=+0.362057878 container start c746712e4c789966ffec1d580a66ef27f4fa034da0d9d1cc8aec983ca5b1bf7b (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0842-4417849) 2024-09-21T12:41:12.727 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:41:12 smithi078 podman[93845]: 2024-09-21 12:41:12.725552445 +0000 UTC m=+0.441818134 container attach c746712e4c789966ffec1d580a66ef27f4fa034da0d9d1cc8aec983ca5b1bf7b (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0842-4417849) 2024-09-21T12:41:13.056 INFO:journalctl@ceph.osd.2.smithi078.stdout:Sep 21 12:41:12 smithi078 podman[93845]: 2024-09-21 12:41:12.83051526 +0000 UTC m=+0.546780962 container died c746712e4c789966ffec1d580a66ef27f4fa034da0d9d1cc8aec983ca5b1bf7b (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-2-deactivate, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0842-4417849) 2024-09-21T12:41:13.445 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.2.service' 2024-09-21T12:41:13.482 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T12:41:13.482 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-21T12:41:13.483 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-21T12:41:13.483 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.3 2024-09-21T12:41:13.806 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:41:13 smithi078 systemd[1]: Stopping Ceph osd.3 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:41:14.306 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:41:13 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3[76270]: 2024-09-21T12:41:13.814+0000 7fc3615d0640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T12:41:14.306 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:41:13 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3[76270]: 2024-09-21T12:41:13.814+0000 7fc3615d0640 -1 osd.3 162 *** Got signal Terminated *** 2024-09-21T12:41:14.306 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:41:13 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3[76270]: 2024-09-21T12:41:13.814+0000 7fc3615d0640 -1 osd.3 162 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T12:41:17.556 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:41:17 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3[76270]: 2024-09-21T12:41:17.226+0000 7fc359bdd640 -1 osd.3 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.922450+0000 front 2024-09-21T12:40:52.922419+0000 (oldest deadline 2024-09-21T12:41:17.022569+0000) 2024-09-21T12:41:17.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:17 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:17.375+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:18.300 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:41:17 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4[53317]: 2024-09-21T12:41:17.865+0000 7f7d78f3a640 -1 osd.4 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.408108+0000 front 2024-09-21T12:40:52.408090+0000 (oldest deadline 2024-09-21T12:41:17.107871+0000) 2024-09-21T12:41:18.300 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:17 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:17.801+0000 7f84132b8640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.243123+0000 front 2024-09-21T12:40:54.243078+0000 (oldest deadline 2024-09-21T12:41:17.743040+0000) 2024-09-21T12:41:18.556 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:41:18 smithi078 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3[76270]: 2024-09-21T12:41:18.244+0000 7fc359bdd640 -1 osd.3 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.922450+0000 front 2024-09-21T12:40:52.922419+0000 (oldest deadline 2024-09-21T12:41:17.022569+0000) 2024-09-21T12:41:18.721 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:18 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:18.362+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:19.050 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:41:18 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4[53317]: 2024-09-21T12:41:18.868+0000 7f7d78f3a640 -1 osd.4 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.408108+0000 front 2024-09-21T12:40:52.408090+0000 (oldest deadline 2024-09-21T12:41:17.107871+0000) 2024-09-21T12:41:19.051 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:18 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:18.834+0000 7f84132b8640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.243123+0000 front 2024-09-21T12:40:54.243078+0000 (oldest deadline 2024-09-21T12:41:17.743040+0000) 2024-09-21T12:41:19.051 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:18 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:18.868+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.510342+0000 front 2024-09-21T12:40:52.510393+0000 (oldest deadline 2024-09-21T12:41:18.410269+0000) 2024-09-21T12:41:19.556 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:41:19 smithi078 podman[93945]: 2024-09-21 12:41:19.275111661 +0000 UTC m=+5.568306110 container died ead87e4306ff0b3cc82222ec6c523a2353649bf817838ad05d2c20f0a263e36c (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-21T12:41:19.666 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:19 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:19.346+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:19.925 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:41:19 smithi078 podman[93945]: 2024-09-21 12:41:19.672857106 +0000 UTC m=+5.966051556 container cleanup ead87e4306ff0b3cc82222ec6c523a2353649bf817838ad05d2c20f0a263e36c (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-21-0842-4417849, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-21T12:41:19.925 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:41:19 smithi078 bash[93945]: ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3 2024-09-21T12:41:20.050 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:19 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:19.871+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.510342+0000 front 2024-09-21T12:40:52.510393+0000 (oldest deadline 2024-09-21T12:41:18.410269+0000) 2024-09-21T12:41:20.051 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:41:19 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4[53317]: 2024-09-21T12:41:19.896+0000 7f7d78f3a640 -1 osd.4 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.408108+0000 front 2024-09-21T12:40:52.408090+0000 (oldest deadline 2024-09-21T12:41:17.107871+0000) 2024-09-21T12:41:20.051 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:19 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:19.802+0000 7f84132b8640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.243123+0000 front 2024-09-21T12:40:54.243078+0000 (oldest deadline 2024-09-21T12:41:17.743040+0000) 2024-09-21T12:41:20.260 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:41:19 smithi078 podman[93961]: 2024-09-21 12:41:19.924129468 +0000 UTC m=+0.638681256 container remove ead87e4306ff0b3cc82222ec6c523a2353649bf817838ad05d2c20f0a263e36c (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-21T12:41:20.556 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:41:20 smithi078 podman[94050]: 2024-09-21 12:41:20.162272657 +0000 UTC m=+0.016307639 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:41:20.557 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:41:20 smithi078 podman[94050]: 2024-09-21 12:41:20.283296021 +0000 UTC m=+0.137330990 container create 7bd7bdf202e7e1b745a1fd7963ad6bdf7370e2a45c1516a2e50b2966c8c1d3a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3-deactivate, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-21T12:41:20.557 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:41:20 smithi078 podman[94050]: 2024-09-21 12:41:20.502526278 +0000 UTC m=+0.356561254 container init 7bd7bdf202e7e1b745a1fd7963ad6bdf7370e2a45c1516a2e50b2966c8c1d3a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0842-4417849, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-21T12:41:20.557 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:41:20 smithi078 podman[94050]: 2024-09-21 12:41:20.505886907 +0000 UTC m=+0.359921874 container start 7bd7bdf202e7e1b745a1fd7963ad6bdf7370e2a45c1516a2e50b2966c8c1d3a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-21-0842-4417849, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True) 2024-09-21T12:41:20.760 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:20 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:20.342+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:21.050 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:20 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:20.759+0000 7f84132b8640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.243123+0000 front 2024-09-21T12:40:54.243078+0000 (oldest deadline 2024-09-21T12:41:17.743040+0000) 2024-09-21T12:41:21.050 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:41:20 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4[53317]: 2024-09-21T12:41:20.864+0000 7f7d78f3a640 -1 osd.4 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.408108+0000 front 2024-09-21T12:40:52.408090+0000 (oldest deadline 2024-09-21T12:41:17.107871+0000) 2024-09-21T12:41:21.051 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:20 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:20.871+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.510342+0000 front 2024-09-21T12:40:52.510393+0000 (oldest deadline 2024-09-21T12:41:18.410269+0000) 2024-09-21T12:41:21.056 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:41:20 smithi078 podman[94050]: 2024-09-21 12:41:20.602467545 +0000 UTC m=+0.456502520 container attach 7bd7bdf202e7e1b745a1fd7963ad6bdf7370e2a45c1516a2e50b2966c8c1d3a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3-deactivate, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T12:41:21.056 INFO:journalctl@ceph.osd.3.smithi078.stdout:Sep 21 12:41:20 smithi078 podman[94050]: 2024-09-21 12:41:20.695659849 +0000 UTC m=+0.549694815 container died 7bd7bdf202e7e1b745a1fd7963ad6bdf7370e2a45c1516a2e50b2966c8c1d3a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-21-0842-4417849, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-21T12:41:21.363 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.3.service' 2024-09-21T12:41:21.401 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T12:41:21.402 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-21T12:41:21.402 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-21T12:41:21.402 DEBUG:teuthology.orchestra.run.smithi148:> sudo systemctl stop ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.4 2024-09-21T12:41:21.720 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:41:21 smithi148 systemd[1]: Stopping Ceph osd.4 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:41:21.720 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:21 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:21.387+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:22.050 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:21 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:21.766+0000 7f84132b8640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.243123+0000 front 2024-09-21T12:40:54.243078+0000 (oldest deadline 2024-09-21T12:41:17.743040+0000) 2024-09-21T12:41:22.051 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:21 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:21.823+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.510342+0000 front 2024-09-21T12:40:52.510393+0000 (oldest deadline 2024-09-21T12:41:18.410269+0000) 2024-09-21T12:41:22.051 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:41:21 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4[53317]: 2024-09-21T12:41:21.718+0000 7f7d7c925640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T12:41:22.051 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:41:21 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4[53317]: 2024-09-21T12:41:21.718+0000 7f7d7c925640 -1 osd.4 162 *** Got signal Terminated *** 2024-09-21T12:41:22.051 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:41:21 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4[53317]: 2024-09-21T12:41:21.718+0000 7f7d7c925640 -1 osd.4 162 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T12:41:22.051 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:41:21 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4[53317]: 2024-09-21T12:41:21.879+0000 7f7d78f3a640 -1 osd.4 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.408108+0000 front 2024-09-21T12:40:52.408090+0000 (oldest deadline 2024-09-21T12:41:17.107871+0000) 2024-09-21T12:41:22.734 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:22 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:22.342+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:23.050 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:22 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:22.732+0000 7f84132b8640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.243123+0000 front 2024-09-21T12:40:54.243078+0000 (oldest deadline 2024-09-21T12:41:17.743040+0000) 2024-09-21T12:41:23.050 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:41:22 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4[53317]: 2024-09-21T12:41:22.905+0000 7f7d78f3a640 -1 osd.4 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.408108+0000 front 2024-09-21T12:40:52.408090+0000 (oldest deadline 2024-09-21T12:41:17.107871+0000) 2024-09-21T12:41:23.051 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:22 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:22.866+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.510342+0000 front 2024-09-21T12:40:52.510393+0000 (oldest deadline 2024-09-21T12:41:18.410269+0000) 2024-09-21T12:41:23.780 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:23 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:23.382+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:24.050 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:41:23 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4[53317]: 2024-09-21T12:41:23.893+0000 7f7d78f3a640 -1 osd.4 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.408108+0000 front 2024-09-21T12:40:52.408090+0000 (oldest deadline 2024-09-21T12:41:17.107871+0000) 2024-09-21T12:41:24.050 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:23 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:23.778+0000 7f84132b8640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.243123+0000 front 2024-09-21T12:40:54.243078+0000 (oldest deadline 2024-09-21T12:41:17.743040+0000) 2024-09-21T12:41:24.051 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:23 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:23.778+0000 7f84132b8640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.444045+0000 front 2024-09-21T12:40:59.444031+0000 (oldest deadline 2024-09-21T12:41:23.444308+0000) 2024-09-21T12:41:24.051 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:23 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:23.827+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.510342+0000 front 2024-09-21T12:40:52.510393+0000 (oldest deadline 2024-09-21T12:41:18.410269+0000) 2024-09-21T12:41:24.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:24 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:24.341+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:25.050 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:41:24 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4[53317]: 2024-09-21T12:41:24.871+0000 7f7d78f3a640 -1 osd.4 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.408108+0000 front 2024-09-21T12:40:52.408090+0000 (oldest deadline 2024-09-21T12:41:17.107871+0000) 2024-09-21T12:41:25.051 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:24 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:24.811+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.510342+0000 front 2024-09-21T12:40:52.510393+0000 (oldest deadline 2024-09-21T12:41:18.410269+0000) 2024-09-21T12:41:25.051 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:24 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:24.798+0000 7f84132b8640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.243123+0000 front 2024-09-21T12:40:54.243078+0000 (oldest deadline 2024-09-21T12:41:17.743040+0000) 2024-09-21T12:41:25.051 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:24 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:24.798+0000 7f84132b8640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.444045+0000 front 2024-09-21T12:40:59.444031+0000 (oldest deadline 2024-09-21T12:41:23.444308+0000) 2024-09-21T12:41:25.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:25 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:25.379+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:25.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:25 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:25.379+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.288524+0000 front 2024-09-21T12:40:59.288493+0000 (oldest deadline 2024-09-21T12:41:25.188392+0000) 2024-09-21T12:41:26.300 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:25 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:25.843+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.510342+0000 front 2024-09-21T12:40:52.510393+0000 (oldest deadline 2024-09-21T12:41:18.410269+0000) 2024-09-21T12:41:26.300 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:25 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:25.842+0000 7f84132b8640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.243123+0000 front 2024-09-21T12:40:54.243078+0000 (oldest deadline 2024-09-21T12:41:17.743040+0000) 2024-09-21T12:41:26.301 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:25 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:25.842+0000 7f84132b8640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.444045+0000 front 2024-09-21T12:40:59.444031+0000 (oldest deadline 2024-09-21T12:41:23.444308+0000) 2024-09-21T12:41:26.301 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:41:25 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4[53317]: 2024-09-21T12:41:25.831+0000 7f7d78f3a640 -1 osd.4 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.408108+0000 front 2024-09-21T12:40:52.408090+0000 (oldest deadline 2024-09-21T12:41:17.107871+0000) 2024-09-21T12:41:26.301 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:41:25 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4[53317]: 2024-09-21T12:41:25.831+0000 7f7d78f3a640 -1 osd.4 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:41:00.509062+0000 front 2024-09-21T12:41:00.509165+0000 (oldest deadline 2024-09-21T12:41:25.209019+0000) 2024-09-21T12:41:26.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:26 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:26.386+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:26.801 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:26 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:26.386+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.288524+0000 front 2024-09-21T12:40:59.288493+0000 (oldest deadline 2024-09-21T12:41:25.188392+0000) 2024-09-21T12:41:27.251 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:26 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:26.884+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.510342+0000 front 2024-09-21T12:40:52.510393+0000 (oldest deadline 2024-09-21T12:41:18.410269+0000) 2024-09-21T12:41:27.252 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:26 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:26.869+0000 7f84132b8640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.243123+0000 front 2024-09-21T12:40:54.243078+0000 (oldest deadline 2024-09-21T12:41:17.743040+0000) 2024-09-21T12:41:27.252 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:26 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:26.869+0000 7f84132b8640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.444045+0000 front 2024-09-21T12:40:59.444031+0000 (oldest deadline 2024-09-21T12:41:23.444308+0000) 2024-09-21T12:41:27.529 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:41:27 smithi148 podman[69121]: 2024-09-21 12:41:27.251038351 +0000 UTC m=+5.642452257 container died 10ec7d6b476e5f74d7548f339428e9fe901dbaea30143793e4ce97383d924d5b (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T12:41:27.529 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:27 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:27.434+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:27.529 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:27 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:27.434+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.288524+0000 front 2024-09-21T12:40:59.288493+0000 (oldest deadline 2024-09-21T12:41:25.188392+0000) 2024-09-21T12:41:27.800 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:41:27 smithi148 podman[69121]: 2024-09-21 12:41:27.71934753 +0000 UTC m=+6.110761439 container cleanup 10ec7d6b476e5f74d7548f339428e9fe901dbaea30143793e4ce97383d924d5b (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-21T12:41:27.800 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:41:27 smithi148 bash[69121]: ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4 2024-09-21T12:41:28.300 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:41:27 smithi148 podman[69135]: 2024-09-21 12:41:27.979342538 +0000 UTC m=+0.731574148 container remove 10ec7d6b476e5f74d7548f339428e9fe901dbaea30143793e4ce97383d924d5b (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-21-0842-4417849, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-21T12:41:28.300 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:27 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:27.877+0000 7f84132b8640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.243123+0000 front 2024-09-21T12:40:54.243078+0000 (oldest deadline 2024-09-21T12:41:17.743040+0000) 2024-09-21T12:41:28.301 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:27 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:27.877+0000 7f84132b8640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.444045+0000 front 2024-09-21T12:40:59.444031+0000 (oldest deadline 2024-09-21T12:41:23.444308+0000) 2024-09-21T12:41:28.301 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:27 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:27.887+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.510342+0000 front 2024-09-21T12:40:52.510393+0000 (oldest deadline 2024-09-21T12:41:18.410269+0000) 2024-09-21T12:41:28.301 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:27 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:27.887+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:41:02.511454+0000 front 2024-09-21T12:41:02.511469+0000 (oldest deadline 2024-09-21T12:41:27.211292+0000) 2024-09-21T12:41:28.658 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:28 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:28.432+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:28.658 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:28 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:28.432+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.288524+0000 front 2024-09-21T12:40:59.288493+0000 (oldest deadline 2024-09-21T12:41:25.188392+0000) 2024-09-21T12:41:28.658 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:41:28 smithi148 podman[69219]: 2024-09-21 12:41:28.214454906 +0000 UTC m=+0.015829105 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:41:28.658 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:41:28 smithi148 podman[69219]: 2024-09-21 12:41:28.339276025 +0000 UTC m=+0.140650227 container create 771e93b7d03cbe5a40bf4ace60836e140f8700124d1c639f8ba3fa600afcf849 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0842-4417849, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T12:41:28.658 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:41:28 smithi148 podman[69219]: 2024-09-21 12:41:28.557445867 +0000 UTC m=+0.358820073 container init 771e93b7d03cbe5a40bf4ace60836e140f8700124d1c639f8ba3fa600afcf849 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-21-0842-4417849, ceph=True) 2024-09-21T12:41:28.659 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:41:28 smithi148 podman[69219]: 2024-09-21 12:41:28.560672783 +0000 UTC m=+0.362046982 container start 771e93b7d03cbe5a40bf4ace60836e140f8700124d1c639f8ba3fa600afcf849 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-21-0842-4417849) 2024-09-21T12:41:28.659 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:41:28 smithi148 podman[69219]: 2024-09-21 12:41:28.657536819 +0000 UTC m=+0.458911020 container attach 771e93b7d03cbe5a40bf4ace60836e140f8700124d1c639f8ba3fa600afcf849 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4-deactivate, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-21T12:41:29.045 INFO:journalctl@ceph.osd.4.smithi148.stdout:Sep 21 12:41:28 smithi148 podman[69219]: 2024-09-21 12:41:28.754968873 +0000 UTC m=+0.556343072 container died 771e93b7d03cbe5a40bf4ace60836e140f8700124d1c639f8ba3fa600afcf849 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-4-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0842-4417849, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True) 2024-09-21T12:41:29.045 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:28 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:28.851+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.510342+0000 front 2024-09-21T12:40:52.510393+0000 (oldest deadline 2024-09-21T12:41:18.410269+0000) 2024-09-21T12:41:29.045 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:28 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:28.851+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:41:02.511454+0000 front 2024-09-21T12:41:02.511469+0000 (oldest deadline 2024-09-21T12:41:27.211292+0000) 2024-09-21T12:41:29.046 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:28 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:28.833+0000 7f84132b8640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.243123+0000 front 2024-09-21T12:40:54.243078+0000 (oldest deadline 2024-09-21T12:41:17.743040+0000) 2024-09-21T12:41:29.046 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:28 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:28.833+0000 7f84132b8640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.444045+0000 front 2024-09-21T12:40:59.444031+0000 (oldest deadline 2024-09-21T12:41:23.444308+0000) 2024-09-21T12:41:29.460 DEBUG:teuthology.orchestra.run.smithi148:> sudo pkill -f 'journalctl -f -n 0 -u ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.4.service' 2024-09-21T12:41:29.499 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T12:41:29.499 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-21T12:41:29.499 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-21T12:41:29.499 DEBUG:teuthology.orchestra.run.smithi148:> sudo systemctl stop ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.5 2024-09-21T12:41:29.742 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:29 smithi148 systemd[1]: Stopping Ceph osd.5 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:41:29.742 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:29 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:29.477+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:29.742 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:29 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:29.477+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.288524+0000 front 2024-09-21T12:40:59.288493+0000 (oldest deadline 2024-09-21T12:41:25.188392+0000) 2024-09-21T12:41:30.050 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:29 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:29.862+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.510342+0000 front 2024-09-21T12:40:52.510393+0000 (oldest deadline 2024-09-21T12:41:18.410269+0000) 2024-09-21T12:41:30.050 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:29 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:29.862+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:41:02.511454+0000 front 2024-09-21T12:41:02.511469+0000 (oldest deadline 2024-09-21T12:41:27.211292+0000) 2024-09-21T12:41:30.050 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:29 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:29.855+0000 7f841acab640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T12:41:30.050 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:29 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:29.855+0000 7f841acab640 -1 osd.5 162 *** Got signal Terminated *** 2024-09-21T12:41:30.051 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:29 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:29.855+0000 7f841acab640 -1 osd.5 162 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T12:41:30.051 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:29 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:29.879+0000 7f84132b8640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.243123+0000 front 2024-09-21T12:40:54.243078+0000 (oldest deadline 2024-09-21T12:41:17.743040+0000) 2024-09-21T12:41:30.051 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:29 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:29.879+0000 7f84132b8640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.444045+0000 front 2024-09-21T12:40:59.444031+0000 (oldest deadline 2024-09-21T12:41:23.444308+0000) 2024-09-21T12:41:30.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:30 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:30.429+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:30.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:30 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:30.429+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.288524+0000 front 2024-09-21T12:40:59.288493+0000 (oldest deadline 2024-09-21T12:41:25.188392+0000) 2024-09-21T12:41:31.300 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:30 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:30.862+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.510342+0000 front 2024-09-21T12:40:52.510393+0000 (oldest deadline 2024-09-21T12:41:18.410269+0000) 2024-09-21T12:41:31.300 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:30 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:30.862+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:41:02.511454+0000 front 2024-09-21T12:41:02.511469+0000 (oldest deadline 2024-09-21T12:41:27.211292+0000) 2024-09-21T12:41:31.301 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:30 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:30.902+0000 7f84132b8640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.243123+0000 front 2024-09-21T12:40:54.243078+0000 (oldest deadline 2024-09-21T12:41:17.743040+0000) 2024-09-21T12:41:31.301 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:30 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:30.902+0000 7f84132b8640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.444045+0000 front 2024-09-21T12:40:59.444031+0000 (oldest deadline 2024-09-21T12:41:23.444308+0000) 2024-09-21T12:41:31.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:31 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:31.429+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:31.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:31 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:31.429+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.288524+0000 front 2024-09-21T12:40:59.288493+0000 (oldest deadline 2024-09-21T12:41:25.188392+0000) 2024-09-21T12:41:32.302 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:31 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:31.852+0000 7f84132b8640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.243123+0000 front 2024-09-21T12:40:54.243078+0000 (oldest deadline 2024-09-21T12:41:17.743040+0000) 2024-09-21T12:41:32.303 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:31 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:31.852+0000 7f84132b8640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.444045+0000 front 2024-09-21T12:40:59.444031+0000 (oldest deadline 2024-09-21T12:41:23.444308+0000) 2024-09-21T12:41:32.303 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:31 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:31.879+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.510342+0000 front 2024-09-21T12:40:52.510393+0000 (oldest deadline 2024-09-21T12:41:18.410269+0000) 2024-09-21T12:41:32.303 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:31 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:31.879+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:41:02.511454+0000 front 2024-09-21T12:41:02.511469+0000 (oldest deadline 2024-09-21T12:41:27.211292+0000) 2024-09-21T12:41:32.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:32 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:32.426+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:32.801 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:32 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:32.426+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.288524+0000 front 2024-09-21T12:40:59.288493+0000 (oldest deadline 2024-09-21T12:41:25.188392+0000) 2024-09-21T12:41:33.300 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:32 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:32.807+0000 7f84132b8640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.243123+0000 front 2024-09-21T12:40:54.243078+0000 (oldest deadline 2024-09-21T12:41:17.743040+0000) 2024-09-21T12:41:33.300 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:32 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:32.807+0000 7f84132b8640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.444045+0000 front 2024-09-21T12:40:59.444031+0000 (oldest deadline 2024-09-21T12:41:23.444308+0000) 2024-09-21T12:41:33.300 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:32 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:32.908+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.510342+0000 front 2024-09-21T12:40:52.510393+0000 (oldest deadline 2024-09-21T12:41:18.410269+0000) 2024-09-21T12:41:33.301 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:32 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:32.908+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:41:02.511454+0000 front 2024-09-21T12:41:02.511469+0000 (oldest deadline 2024-09-21T12:41:27.211292+0000) 2024-09-21T12:41:33.301 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:32 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:32.908+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-21T12:41:10.712185+0000 front 2024-09-21T12:41:10.712224+0000 (oldest deadline 2024-09-21T12:41:32.412139+0000) 2024-09-21T12:41:33.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:33 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:33.462+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:33.801 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:33 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:33.462+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.288524+0000 front 2024-09-21T12:40:59.288493+0000 (oldest deadline 2024-09-21T12:41:25.188392+0000) 2024-09-21T12:41:33.801 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:33 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:33.462+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-21T12:41:09.789867+0000 front 2024-09-21T12:41:09.789788+0000 (oldest deadline 2024-09-21T12:41:32.689778+0000) 2024-09-21T12:41:34.300 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:33 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:33.820+0000 7f84132b8640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.243123+0000 front 2024-09-21T12:40:54.243078+0000 (oldest deadline 2024-09-21T12:41:17.743040+0000) 2024-09-21T12:41:34.300 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:33 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:33.820+0000 7f84132b8640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.444045+0000 front 2024-09-21T12:40:59.444031+0000 (oldest deadline 2024-09-21T12:41:23.444308+0000) 2024-09-21T12:41:34.300 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:33 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:33.868+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.510342+0000 front 2024-09-21T12:40:52.510393+0000 (oldest deadline 2024-09-21T12:41:18.410269+0000) 2024-09-21T12:41:34.301 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:33 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:33.868+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:41:02.511454+0000 front 2024-09-21T12:41:02.511469+0000 (oldest deadline 2024-09-21T12:41:27.211292+0000) 2024-09-21T12:41:34.301 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:33 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:33.868+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-21T12:41:10.712185+0000 front 2024-09-21T12:41:10.712224+0000 (oldest deadline 2024-09-21T12:41:32.412139+0000) 2024-09-21T12:41:34.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:34 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:34.459+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:34.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:34 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:34.459+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.288524+0000 front 2024-09-21T12:40:59.288493+0000 (oldest deadline 2024-09-21T12:41:25.188392+0000) 2024-09-21T12:41:34.801 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:34 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:34.459+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-21T12:41:09.789867+0000 front 2024-09-21T12:41:09.789788+0000 (oldest deadline 2024-09-21T12:41:32.689778+0000) 2024-09-21T12:41:35.279 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:34 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:34.914+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.510342+0000 front 2024-09-21T12:40:52.510393+0000 (oldest deadline 2024-09-21T12:41:18.410269+0000) 2024-09-21T12:41:35.280 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:34 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:34.914+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:41:02.511454+0000 front 2024-09-21T12:41:02.511469+0000 (oldest deadline 2024-09-21T12:41:27.211292+0000) 2024-09-21T12:41:35.280 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:34 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:34.914+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-21T12:41:10.712185+0000 front 2024-09-21T12:41:10.712224+0000 (oldest deadline 2024-09-21T12:41:32.412139+0000) 2024-09-21T12:41:35.280 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:34 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:34.832+0000 7f84132b8640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.243123+0000 front 2024-09-21T12:40:54.243078+0000 (oldest deadline 2024-09-21T12:41:17.743040+0000) 2024-09-21T12:41:35.280 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:34 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:34.832+0000 7f84132b8640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.444045+0000 front 2024-09-21T12:40:59.444031+0000 (oldest deadline 2024-09-21T12:41:23.444308+0000) 2024-09-21T12:41:35.280 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:34 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5[55401]: 2024-09-21T12:41:34.832+0000 7f84132b8640 -1 osd.5 162 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-21T12:41:08.045764+0000 front 2024-09-21T12:41:08.045779+0000 (oldest deadline 2024-09-21T12:41:33.945674+0000) 2024-09-21T12:41:35.549 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:35 smithi148 podman[69321]: 2024-09-21 12:41:35.279084147 +0000 UTC m=+5.557743623 container died 06ac02700ea142595989cba71ee709466fa5661cf7cdc5175bd2baba234062a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-21-0842-4417849, maintainer=Guillaume Abrioux ) 2024-09-21T12:41:35.550 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:35 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:35.494+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:35.550 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:35 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:35.494+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.288524+0000 front 2024-09-21T12:40:59.288493+0000 (oldest deadline 2024-09-21T12:41:25.188392+0000) 2024-09-21T12:41:35.550 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:35 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:35.494+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-21T12:41:09.789867+0000 front 2024-09-21T12:41:09.789788+0000 (oldest deadline 2024-09-21T12:41:32.689778+0000) 2024-09-21T12:41:35.800 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:35 smithi148 podman[69321]: 2024-09-21 12:41:35.731499612 +0000 UTC m=+6.010159082 container cleanup 06ac02700ea142595989cba71ee709466fa5661cf7cdc5175bd2baba234062a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-21T12:41:35.800 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:35 smithi148 bash[69321]: ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5 2024-09-21T12:41:36.300 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:36 smithi148 podman[69351]: 2024-09-21 12:41:36.007840555 +0000 UTC m=+0.725282824 container remove 06ac02700ea142595989cba71ee709466fa5661cf7cdc5175bd2baba234062a5 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-21T12:41:36.300 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:35 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:35.950+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.510342+0000 front 2024-09-21T12:40:52.510393+0000 (oldest deadline 2024-09-21T12:41:18.410269+0000) 2024-09-21T12:41:36.301 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:35 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:35.950+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:41:02.511454+0000 front 2024-09-21T12:41:02.511469+0000 (oldest deadline 2024-09-21T12:41:27.211292+0000) 2024-09-21T12:41:36.301 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:35 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:35.950+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-21T12:41:10.712185+0000 front 2024-09-21T12:41:10.712224+0000 (oldest deadline 2024-09-21T12:41:32.412139+0000) 2024-09-21T12:41:36.653 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:36 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:36.519+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:36.653 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:36 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:36.519+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.288524+0000 front 2024-09-21T12:40:59.288493+0000 (oldest deadline 2024-09-21T12:41:25.188392+0000) 2024-09-21T12:41:36.653 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:36 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:36.519+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-21T12:41:09.789867+0000 front 2024-09-21T12:41:09.789788+0000 (oldest deadline 2024-09-21T12:41:32.689778+0000) 2024-09-21T12:41:36.653 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:36 smithi148 podman[69439]: 2024-09-21 12:41:36.243144362 +0000 UTC m=+0.016674026 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:41:36.653 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:36 smithi148 podman[69439]: 2024-09-21 12:41:36.341917089 +0000 UTC m=+0.115446796 container create 7a745e8a6652d0d9da83ba443eaad2ef8e63b33cd04f991687528ebd4efe3112 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5-deactivate, RELEASE=wip-guits-main-2024-09-21-0842-4417849, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-21T12:41:36.653 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:36 smithi148 podman[69439]: 2024-09-21 12:41:36.569129345 +0000 UTC m=+0.342659009 container init 7a745e8a6652d0d9da83ba443eaad2ef8e63b33cd04f991687528ebd4efe3112 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True) 2024-09-21T12:41:36.654 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:36 smithi148 podman[69439]: 2024-09-21 12:41:36.572485639 +0000 UTC m=+0.346015314 container start 7a745e8a6652d0d9da83ba443eaad2ef8e63b33cd04f991687528ebd4efe3112 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T12:41:36.654 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:36 smithi148 podman[69439]: 2024-09-21 12:41:36.652452119 +0000 UTC m=+0.425981793 container attach 7a745e8a6652d0d9da83ba443eaad2ef8e63b33cd04f991687528ebd4efe3112 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5-deactivate, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T12:41:36.964 INFO:journalctl@ceph.osd.5.smithi148.stdout:Sep 21 12:41:36 smithi148 podman[69439]: 2024-09-21 12:41:36.760834376 +0000 UTC m=+0.534364046 container died 7a745e8a6652d0d9da83ba443eaad2ef8e63b33cd04f991687528ebd4efe3112 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-5-deactivate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0842-4417849, io.buildah.version=1.37.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-21T12:41:37.300 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:36 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:36.963+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.510342+0000 front 2024-09-21T12:40:52.510393+0000 (oldest deadline 2024-09-21T12:41:18.410269+0000) 2024-09-21T12:41:37.300 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:36 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:36.963+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:41:02.511454+0000 front 2024-09-21T12:41:02.511469+0000 (oldest deadline 2024-09-21T12:41:27.211292+0000) 2024-09-21T12:41:37.300 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:36 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:36.963+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-21T12:41:10.712185+0000 front 2024-09-21T12:41:10.712224+0000 (oldest deadline 2024-09-21T12:41:32.412139+0000) 2024-09-21T12:41:37.472 DEBUG:teuthology.orchestra.run.smithi148:> sudo pkill -f 'journalctl -f -n 0 -u ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.5.service' 2024-09-21T12:41:37.511 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T12:41:37.511 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-21T12:41:37.512 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-21T12:41:37.512 DEBUG:teuthology.orchestra.run.smithi148:> sudo systemctl stop ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.6 2024-09-21T12:41:37.800 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:37 smithi148 systemd[1]: Stopping Ceph osd.6 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:41:37.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:37 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:37.471+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:37.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:37 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:37.471+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.288524+0000 front 2024-09-21T12:40:59.288493+0000 (oldest deadline 2024-09-21T12:41:25.188392+0000) 2024-09-21T12:41:37.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:37 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:37.471+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-21T12:41:09.789867+0000 front 2024-09-21T12:41:09.789788+0000 (oldest deadline 2024-09-21T12:41:32.689778+0000) 2024-09-21T12:41:38.300 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:37 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:37.860+0000 7f560af02640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T12:41:38.300 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:37 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:37.860+0000 7f560af02640 -1 osd.6 162 *** Got signal Terminated *** 2024-09-21T12:41:38.301 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:37 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:37.860+0000 7f560af02640 -1 osd.6 162 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T12:41:38.301 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:37 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:37.976+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.510342+0000 front 2024-09-21T12:40:52.510393+0000 (oldest deadline 2024-09-21T12:41:18.410269+0000) 2024-09-21T12:41:38.301 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:37 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:37.976+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:41:02.511454+0000 front 2024-09-21T12:41:02.511469+0000 (oldest deadline 2024-09-21T12:41:27.211292+0000) 2024-09-21T12:41:38.301 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:37 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:37.976+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-21T12:41:10.712185+0000 front 2024-09-21T12:41:10.712224+0000 (oldest deadline 2024-09-21T12:41:32.412139+0000) 2024-09-21T12:41:38.801 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:38 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:38.519+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:38.801 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:38 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:38.519+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.288524+0000 front 2024-09-21T12:40:59.288493+0000 (oldest deadline 2024-09-21T12:41:25.188392+0000) 2024-09-21T12:41:38.801 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:38 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:38.519+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-21T12:41:09.789867+0000 front 2024-09-21T12:41:09.789788+0000 (oldest deadline 2024-09-21T12:41:32.689778+0000) 2024-09-21T12:41:39.300 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:38 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:38.940+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.510342+0000 front 2024-09-21T12:40:52.510393+0000 (oldest deadline 2024-09-21T12:41:18.410269+0000) 2024-09-21T12:41:39.300 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:38 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:38.940+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:41:02.511454+0000 front 2024-09-21T12:41:02.511469+0000 (oldest deadline 2024-09-21T12:41:27.211292+0000) 2024-09-21T12:41:39.301 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:38 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:38.940+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-21T12:41:10.712185+0000 front 2024-09-21T12:41:10.712224+0000 (oldest deadline 2024-09-21T12:41:32.412139+0000) 2024-09-21T12:41:39.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:39 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:39.492+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:39.801 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:39 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:39.492+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.288524+0000 front 2024-09-21T12:40:59.288493+0000 (oldest deadline 2024-09-21T12:41:25.188392+0000) 2024-09-21T12:41:39.801 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:39 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:39.492+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-21T12:41:09.789867+0000 front 2024-09-21T12:41:09.789788+0000 (oldest deadline 2024-09-21T12:41:32.689778+0000) 2024-09-21T12:41:40.300 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:39 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:39.956+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.510342+0000 front 2024-09-21T12:40:52.510393+0000 (oldest deadline 2024-09-21T12:41:18.410269+0000) 2024-09-21T12:41:40.300 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:39 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:39.956+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:41:02.511454+0000 front 2024-09-21T12:41:02.511469+0000 (oldest deadline 2024-09-21T12:41:27.211292+0000) 2024-09-21T12:41:40.300 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:39 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:39.956+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-21T12:41:10.712185+0000 front 2024-09-21T12:41:10.712224+0000 (oldest deadline 2024-09-21T12:41:32.412139+0000) 2024-09-21T12:41:40.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:40 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:40.535+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:40.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:40 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:40.535+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.288524+0000 front 2024-09-21T12:40:59.288493+0000 (oldest deadline 2024-09-21T12:41:25.188392+0000) 2024-09-21T12:41:40.801 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:40 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:40.535+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-21T12:41:09.789867+0000 front 2024-09-21T12:41:09.789788+0000 (oldest deadline 2024-09-21T12:41:32.689778+0000) 2024-09-21T12:41:40.801 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:40 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:40.535+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-21T12:41:16.591456+0000 front 2024-09-21T12:41:16.591441+0000 (oldest deadline 2024-09-21T12:41:40.091346+0000) 2024-09-21T12:41:41.300 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:40 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:40.970+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.510342+0000 front 2024-09-21T12:40:52.510393+0000 (oldest deadline 2024-09-21T12:41:18.410269+0000) 2024-09-21T12:41:41.300 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:40 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:40.970+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:41:02.511454+0000 front 2024-09-21T12:41:02.511469+0000 (oldest deadline 2024-09-21T12:41:27.211292+0000) 2024-09-21T12:41:41.300 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:40 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:40.970+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-21T12:41:10.712185+0000 front 2024-09-21T12:41:10.712224+0000 (oldest deadline 2024-09-21T12:41:32.412139+0000) 2024-09-21T12:41:42.015 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:41 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:41.581+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:42.015 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:41 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:41.581+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.288524+0000 front 2024-09-21T12:40:59.288493+0000 (oldest deadline 2024-09-21T12:41:25.188392+0000) 2024-09-21T12:41:42.015 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:41 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:41.581+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-21T12:41:09.789867+0000 front 2024-09-21T12:41:09.789788+0000 (oldest deadline 2024-09-21T12:41:32.689778+0000) 2024-09-21T12:41:42.015 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:41 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:41.581+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-21T12:41:16.591456+0000 front 2024-09-21T12:41:16.591441+0000 (oldest deadline 2024-09-21T12:41:40.091346+0000) 2024-09-21T12:41:42.300 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:42 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:42.013+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:52.510342+0000 front 2024-09-21T12:40:52.510393+0000 (oldest deadline 2024-09-21T12:41:18.410269+0000) 2024-09-21T12:41:42.300 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:42 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:42.013+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:41:02.511454+0000 front 2024-09-21T12:41:02.511469+0000 (oldest deadline 2024-09-21T12:41:27.211292+0000) 2024-09-21T12:41:42.300 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:42 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6[57512]: 2024-09-21T12:41:42.013+0000 7f5607517640 -1 osd.6 162 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-21T12:41:10.712185+0000 front 2024-09-21T12:41:10.712224+0000 (oldest deadline 2024-09-21T12:41:32.412139+0000) 2024-09-21T12:41:42.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:42 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:42.532+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:42.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:42 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:42.532+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.288524+0000 front 2024-09-21T12:40:59.288493+0000 (oldest deadline 2024-09-21T12:41:25.188392+0000) 2024-09-21T12:41:42.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:42 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:42.532+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-21T12:41:09.789867+0000 front 2024-09-21T12:41:09.789788+0000 (oldest deadline 2024-09-21T12:41:32.689778+0000) 2024-09-21T12:41:42.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:42 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:42.532+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-21T12:41:16.591456+0000 front 2024-09-21T12:41:16.591441+0000 (oldest deadline 2024-09-21T12:41:40.091346+0000) 2024-09-21T12:41:43.734 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:43 smithi148 podman[69540]: 2024-09-21 12:41:43.437945367 +0000 UTC m=+5.695138141 container died 17613684a6534c4b14e1248a618ddf07b88a8c1f0a73a75f11ef7202d6173094 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T12:41:43.735 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:43 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:43.526+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:43.735 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:43 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:43.526+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.288524+0000 front 2024-09-21T12:40:59.288493+0000 (oldest deadline 2024-09-21T12:41:25.188392+0000) 2024-09-21T12:41:43.735 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:43 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:43.526+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-21T12:41:09.789867+0000 front 2024-09-21T12:41:09.789788+0000 (oldest deadline 2024-09-21T12:41:32.689778+0000) 2024-09-21T12:41:43.735 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:43 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:43.526+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-21T12:41:16.591456+0000 front 2024-09-21T12:41:16.591441+0000 (oldest deadline 2024-09-21T12:41:40.091346+0000) 2024-09-21T12:41:44.050 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:43 smithi148 podman[69540]: 2024-09-21 12:41:43.832043083 +0000 UTC m=+6.089235856 container cleanup 17613684a6534c4b14e1248a618ddf07b88a8c1f0a73a75f11ef7202d6173094 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-21-0842-4417849, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-21T12:41:44.050 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:43 smithi148 bash[69540]: ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6 2024-09-21T12:41:44.430 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:44 smithi148 podman[69556]: 2024-09-21 12:41:44.110621165 +0000 UTC m=+0.670597658 container remove 17613684a6534c4b14e1248a618ddf07b88a8c1f0a73a75f11ef7202d6173094 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T12:41:44.799 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:44 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:44.531+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:44.799 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:44 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:44.531+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.288524+0000 front 2024-09-21T12:40:59.288493+0000 (oldest deadline 2024-09-21T12:41:25.188392+0000) 2024-09-21T12:41:44.799 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:44 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:44.531+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-21T12:41:09.789867+0000 front 2024-09-21T12:41:09.789788+0000 (oldest deadline 2024-09-21T12:41:32.689778+0000) 2024-09-21T12:41:44.799 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:44 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:44.531+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-21T12:41:16.591456+0000 front 2024-09-21T12:41:16.591441+0000 (oldest deadline 2024-09-21T12:41:40.091346+0000) 2024-09-21T12:41:44.800 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:44 smithi148 podman[69645]: 2024-09-21 12:41:44.332244627 +0000 UTC m=+0.015830235 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:41:44.800 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:44 smithi148 podman[69645]: 2024-09-21 12:41:44.446207141 +0000 UTC m=+0.129792744 container create 3737798f299db1599f3c0c379a93c8efb46c2375c4e8f08b7b105b7b08d22d65 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6-deactivate, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True) 2024-09-21T12:41:44.800 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:44 smithi148 podman[69645]: 2024-09-21 12:41:44.673584016 +0000 UTC m=+0.357169624 container init 3737798f299db1599f3c0c379a93c8efb46c2375c4e8f08b7b105b7b08d22d65 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731) 2024-09-21T12:41:44.800 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:44 smithi148 podman[69645]: 2024-09-21 12:41:44.676943006 +0000 UTC m=+0.360528610 container start 3737798f299db1599f3c0c379a93c8efb46c2375c4e8f08b7b105b7b08d22d65 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-21-0842-4417849, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-21T12:41:44.800 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:44 smithi148 podman[69645]: 2024-09-21 12:41:44.79879867 +0000 UTC m=+0.482384273 container attach 3737798f299db1599f3c0c379a93c8efb46c2375c4e8f08b7b105b7b08d22d65 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6-deactivate, RELEASE=wip-guits-main-2024-09-21-0842-4417849, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-21T12:41:45.050 INFO:journalctl@ceph.osd.6.smithi148.stdout:Sep 21 12:41:44 smithi148 podman[69645]: 2024-09-21 12:41:44.86298194 +0000 UTC m=+0.546567544 container died 3737798f299db1599f3c0c379a93c8efb46c2375c4e8f08b7b105b7b08d22d65 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-6-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-21-0842-4417849, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-21T12:41:45.473 DEBUG:teuthology.orchestra.run.smithi148:> sudo pkill -f 'journalctl -f -n 0 -u ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.6.service' 2024-09-21T12:41:45.510 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T12:41:45.510 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-21T12:41:45.511 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-21T12:41:45.511 DEBUG:teuthology.orchestra.run.smithi148:> sudo systemctl stop ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.7 2024-09-21T12:41:45.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:45 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:45.580+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:45.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:45 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:45.580+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.288524+0000 front 2024-09-21T12:40:59.288493+0000 (oldest deadline 2024-09-21T12:41:25.188392+0000) 2024-09-21T12:41:45.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:45 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:45.580+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-21T12:41:09.789867+0000 front 2024-09-21T12:41:09.789788+0000 (oldest deadline 2024-09-21T12:41:32.689778+0000) 2024-09-21T12:41:45.801 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:45 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:45.580+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-21T12:41:16.591456+0000 front 2024-09-21T12:41:16.591441+0000 (oldest deadline 2024-09-21T12:41:40.091346+0000) 2024-09-21T12:41:45.801 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:45 smithi148 systemd[1]: Stopping Ceph osd.7 for 6263deca-7813-11ef-baf6-efdc52797490... 2024-09-21T12:41:46.300 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:45 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:45.816+0000 7fb9a6841640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T12:41:46.300 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:45 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:45.816+0000 7fb9a6841640 -1 osd.7 162 *** Got signal Terminated *** 2024-09-21T12:41:46.300 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:45 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:45.816+0000 7fb9a6841640 -1 osd.7 162 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T12:41:46.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:46 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:46.548+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:46.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:46 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:46.548+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.288524+0000 front 2024-09-21T12:40:59.288493+0000 (oldest deadline 2024-09-21T12:41:25.188392+0000) 2024-09-21T12:41:46.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:46 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:46.548+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-21T12:41:09.789867+0000 front 2024-09-21T12:41:09.789788+0000 (oldest deadline 2024-09-21T12:41:32.689778+0000) 2024-09-21T12:41:46.800 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:46 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:46.548+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-21T12:41:16.591456+0000 front 2024-09-21T12:41:16.591441+0000 (oldest deadline 2024-09-21T12:41:40.091346+0000) 2024-09-21T12:41:47.956 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:47 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:47.580+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:47.956 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:47 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:47.580+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.288524+0000 front 2024-09-21T12:40:59.288493+0000 (oldest deadline 2024-09-21T12:41:25.188392+0000) 2024-09-21T12:41:47.956 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:47 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:47.580+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-21T12:41:09.789867+0000 front 2024-09-21T12:41:09.789788+0000 (oldest deadline 2024-09-21T12:41:32.689778+0000) 2024-09-21T12:41:47.956 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:47 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:47.580+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-21T12:41:16.591456+0000 front 2024-09-21T12:41:16.591441+0000 (oldest deadline 2024-09-21T12:41:40.091346+0000) 2024-09-21T12:41:48.897 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:48 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:48.570+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:48.897 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:48 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:48.570+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.288524+0000 front 2024-09-21T12:40:59.288493+0000 (oldest deadline 2024-09-21T12:41:25.188392+0000) 2024-09-21T12:41:48.897 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:48 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:48.570+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-21T12:41:09.789867+0000 front 2024-09-21T12:41:09.789788+0000 (oldest deadline 2024-09-21T12:41:32.689778+0000) 2024-09-21T12:41:48.897 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:48 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:48.570+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-21T12:41:16.591456+0000 front 2024-09-21T12:41:16.591441+0000 (oldest deadline 2024-09-21T12:41:40.091346+0000) 2024-09-21T12:41:48.897 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:48 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:48.570+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.148:6804 osd.4 since back 2024-09-21T12:41:22.391985+0000 front 2024-09-21T12:41:22.392069+0000 (oldest deadline 2024-09-21T12:41:48.292076+0000) 2024-09-21T12:41:50.050 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:49 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:49.584+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:50.050 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:49 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:49.584+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.288524+0000 front 2024-09-21T12:40:59.288493+0000 (oldest deadline 2024-09-21T12:41:25.188392+0000) 2024-09-21T12:41:50.051 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:49 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:49.584+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-21T12:41:09.789867+0000 front 2024-09-21T12:41:09.789788+0000 (oldest deadline 2024-09-21T12:41:32.689778+0000) 2024-09-21T12:41:50.051 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:49 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:49.584+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-21T12:41:16.591456+0000 front 2024-09-21T12:41:16.591441+0000 (oldest deadline 2024-09-21T12:41:40.091346+0000) 2024-09-21T12:41:50.051 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:49 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:49.584+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.148:6804 osd.4 since back 2024-09-21T12:41:22.391985+0000 front 2024-09-21T12:41:22.392069+0000 (oldest deadline 2024-09-21T12:41:48.292076+0000) 2024-09-21T12:41:51.050 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:50 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:50.595+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6806 osd.0 since back 2024-09-21T12:40:54.687423+0000 front 2024-09-21T12:40:54.687434+0000 (oldest deadline 2024-09-21T12:41:16.987309+0000) 2024-09-21T12:41:51.050 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:50 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:50.595+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6814 osd.1 since back 2024-09-21T12:40:59.288524+0000 front 2024-09-21T12:40:59.288493+0000 (oldest deadline 2024-09-21T12:41:25.188392+0000) 2024-09-21T12:41:51.051 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:50 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:50.595+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6822 osd.2 since back 2024-09-21T12:41:09.789867+0000 front 2024-09-21T12:41:09.789788+0000 (oldest deadline 2024-09-21T12:41:32.689778+0000) 2024-09-21T12:41:51.051 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:50 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:50.596+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.78:6830 osd.3 since back 2024-09-21T12:41:16.591456+0000 front 2024-09-21T12:41:16.591441+0000 (oldest deadline 2024-09-21T12:41:40.091346+0000) 2024-09-21T12:41:51.051 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:50 smithi148 ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7[59616]: 2024-09-21T12:41:50.596+0000 7fb99ee4e640 -1 osd.7 162 heartbeat_check: no reply from 172.21.15.148:6804 osd.4 since back 2024-09-21T12:41:22.391985+0000 front 2024-09-21T12:41:22.392069+0000 (oldest deadline 2024-09-21T12:41:48.292076+0000) 2024-09-21T12:41:51.679 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:51 smithi148 podman[69749]: 2024-09-21 12:41:51.403594897 +0000 UTC m=+5.678152905 container died 2e6a6f842c47f317e3b177da026caf78ade2f25a8f4925cf9aa84f2f7d13f7c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-21-0842-4417849, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-21T12:41:52.050 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:51 smithi148 podman[69749]: 2024-09-21 12:41:51.779245383 +0000 UTC m=+6.053803398 container cleanup 2e6a6f842c47f317e3b177da026caf78ade2f25a8f4925cf9aa84f2f7d13f7c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0842-4417849, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-21T12:41:52.050 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:51 smithi148 bash[69749]: ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7 2024-09-21T12:41:52.397 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:52 smithi148 podman[69777]: 2024-09-21 12:41:52.051350638 +0000 UTC m=+0.649802572 container remove 2e6a6f842c47f317e3b177da026caf78ade2f25a8f4925cf9aa84f2f7d13f7c0 (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-21-0842-4417849, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-21T12:41:52.719 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:52 smithi148 podman[69883]: 2024-09-21 12:41:52.299233118 +0000 UTC m=+0.015506844 image pull b415816db8b41f544cdfdc71d193ff8b1ee008bf7b0ec4f64da8eb8f71b62e85 quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5 2024-09-21T12:41:52.719 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:52 smithi148 podman[69883]: 2024-09-21 12:41:52.41216976 +0000 UTC m=+0.128443471 container create 737527c96fff1d0e1b2317232251bea17b0068551a8a948487674d60668eda9c (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7-deactivate, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0842-4417849, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T12:41:52.719 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:52 smithi148 podman[69883]: 2024-09-21 12:41:52.633379767 +0000 UTC m=+0.349653485 container init 737527c96fff1d0e1b2317232251bea17b0068551a8a948487674d60668eda9c (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-21T12:41:52.719 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:52 smithi148 podman[69883]: 2024-09-21 12:41:52.636630446 +0000 UTC m=+0.352904158 container start 737527c96fff1d0e1b2317232251bea17b0068551a8a948487674d60668eda9c (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-21T12:41:53.050 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:52 smithi148 podman[69883]: 2024-09-21 12:41:52.717702501 +0000 UTC m=+0.433976231 container attach 737527c96fff1d0e1b2317232251bea17b0068551a8a948487674d60668eda9c (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-21-0842-4417849, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-21T12:41:53.050 INFO:journalctl@ceph.osd.7.smithi148.stdout:Sep 21 12:41:52 smithi148 podman[69883]: 2024-09-21 12:41:52.830096115 +0000 UTC m=+0.546369827 container died 737527c96fff1d0e1b2317232251bea17b0068551a8a948487674d60668eda9c (image=quay.ceph.io/ceph-ci/ceph@sha256:f8fbbbcab363df772d74f326e361db3b4f83aa3233da1ebe9de08a83e748d9f5, name=ceph-6263deca-7813-11ef-baf6-efdc52797490-osd-7-deactivate, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-21-0842-4417849, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-21T12:41:53.484 DEBUG:teuthology.orchestra.run.smithi148:> sudo pkill -f 'journalctl -f -n 0 -u ceph-6263deca-7813-11ef-baf6-efdc52797490@osd.7.service' 2024-09-21T12:41:53.523 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T12:41:53.523 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-21T12:41:53.524 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 6263deca-7813-11ef-baf6-efdc52797490 --force --keep-logs 2024-09-21T12:44:49.720 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 6263deca-7813-11ef-baf6-efdc52797490 --force --keep-logs 2024-09-21T12:45:10.340 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T12:45:10.373 DEBUG:teuthology.orchestra.run.smithi148:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T12:45:10.400 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-21T12:45:10.402 DEBUG:teuthology.misc:Transferring archived files from smithi078:/var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/crash to /home/teuthworker/archive/gabrioux-2024-09-21_11:23:40-orch:cephadm-wip-guits-main-2024-09-21-0842-distro-default-smithi/7913924/remote/smithi078/crash 2024-09-21T12:45:10.402 DEBUG:teuthology.orchestra.run.smithi078:> sudo tar c -f - -C /var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/crash -- . 2024-09-21T12:45:10.451 INFO:teuthology.orchestra.run.smithi078.stderr:tar: /var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/crash: Cannot open: No such file or directory 2024-09-21T12:45:10.452 INFO:teuthology.orchestra.run.smithi078.stderr:tar: Error is not recoverable: exiting now 2024-09-21T12:45:10.453 DEBUG:teuthology.misc:Transferring archived files from smithi148:/var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/crash to /home/teuthworker/archive/gabrioux-2024-09-21_11:23:40-orch:cephadm-wip-guits-main-2024-09-21-0842-distro-default-smithi/7913924/remote/smithi148/crash 2024-09-21T12:45:10.454 DEBUG:teuthology.orchestra.run.smithi148:> sudo tar c -f - -C /var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/crash -- . 2024-09-21T12:45:10.492 INFO:teuthology.orchestra.run.smithi148.stderr:tar: /var/lib/ceph/6263deca-7813-11ef-baf6-efdc52797490/crash: Cannot open: No such file or directory 2024-09-21T12:45:10.492 INFO:teuthology.orchestra.run.smithi148.stderr:tar: Error is not recoverable: exiting now 2024-09-21T12:45:10.493 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-21T12:45:10.493 DEBUG:teuthology.orchestra.run.smithi078:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v CEPHADM_REFRESH_FAILED | head -n 1 2024-09-21T12:45:10.523 INFO:tasks.cephadm:Compressing logs... 2024-09-21T12:45:10.523 DEBUG:teuthology.orchestra.run.smithi078:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-21T12:45:10.567 DEBUG:teuthology.orchestra.run.smithi148:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-21T12:45:10.590 INFO:teuthology.orchestra.run.smithi078.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-osd.0.log 2024-09-21T12:45:10.590 INFO:teuthology.orchestra.run.smithi078.stderr:: No such file or directory 2024-09-21T12:45:10.590 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-osd.1.log 2024-09-21T12:45:10.592 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-volume.log 2024-09-21T12:45:10.592 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-osd.0.log: /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-mon.smithi078.log 2024-09-21T12:45:10.592 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph.audit.log 2024-09-21T12:45:10.592 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph.cephadm.log 2024-09-21T12:45:10.592 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-mon.smithi078.log: /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-mgr.smithi078.ntwyan.log 2024-09-21T12:45:10.593 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-osd.2.log 2024-09-21T12:45:10.593 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-mgr.smithi078.ntwyan.log: gzip -5 --verbose -- /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph.log 2024-09-21T12:45:10.594 INFO:teuthology.orchestra.run.smithi078.stderr: 85.7% -- replaced with /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph.cephadm.log.gz 2024-09-21T12:45:10.594 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-mds.foofs.smithi078.ytpzwh.log 2024-09-21T12:45:10.595 INFO:teuthology.orchestra.run.smithi148.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-osd.7.log 2024-09-21T12:45:10.595 INFO:teuthology.orchestra.run.smithi148.stderr:: No such file or directory 2024-09-21T12:45:10.595 INFO:teuthology.orchestra.run.smithi148.stderr:gzip -5 --verbose -- /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-volume.log 2024-09-21T12:45:10.595 INFO:teuthology.orchestra.run.smithi148.stderr:gzip -5 --verbose -- /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-osd.4.log 2024-09-21T12:45:10.595 INFO:teuthology.orchestra.run.smithi148.stderr:/var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-osd.7.log: /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-mgr.smithi148.oyiqmx.log 2024-09-21T12:45:10.595 INFO:teuthology.orchestra.run.smithi148.stderr:/var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph.audit.log 2024-09-21T12:45:10.599 INFO:teuthology.orchestra.run.smithi148.stderr:gzip -5 --verbose -- /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-mds.foofs.smithi148.uukezr.log 2024-09-21T12:45:10.599 INFO:teuthology.orchestra.run.smithi148.stderr:/var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-mgr.smithi148.oyiqmx.log: /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-mon.smithi148.log 2024-09-21T12:45:10.599 INFO:teuthology.orchestra.run.smithi148.stderr:/var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-mds.foofs.smithi148.uukezr.log: gzip -5 --verbose -- /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph.cephadm.log 2024-09-21T12:45:10.599 INFO:teuthology.orchestra.run.smithi148.stderr:/var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-mon.smithi148.log: 88.5%gzip -5 --verbose -- /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-osd.6.log 2024-09-21T12:45:10.599 INFO:teuthology.orchestra.run.smithi148.stderr: -- replaced with /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-mds.foofs.smithi148.uukezr.log.gz 2024-09-21T12:45:10.599 INFO:teuthology.orchestra.run.smithi148.stderr:/var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph.cephadm.log: 85.7% -- replaced with /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph.cephadm.log.gz 2024-09-21T12:45:10.599 INFO:teuthology.orchestra.run.smithi148.stderr:gzip -5 --verbose -- /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-osd.5.log 2024-09-21T12:45:10.599 INFO:teuthology.orchestra.run.smithi148.stderr:/var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph.log 2024-09-21T12:45:10.600 INFO:teuthology.orchestra.run.smithi148.stderr:/var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-osd.5.log: 91.6% -- replaced with /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph.audit.log.gz 2024-09-21T12:45:10.600 INFO:teuthology.orchestra.run.smithi148.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-21T12:45:10.601 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph.log: 91.5% -- replaced with /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph.audit.log.gz 2024-09-21T12:45:10.601 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-osd.3.log 2024-09-21T12:45:10.601 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-mds.foofs.smithi078.ytpzwh.log: 87.2% -- replaced with /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph.log.gz 2024-09-21T12:45:10.601 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-21T12:45:10.602 INFO:teuthology.orchestra.run.smithi148.stderr:/var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph.log: 87.2% -- replaced with /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph.log.gz 2024-09-21T12:45:10.609 INFO:teuthology.orchestra.run.smithi148.stderr:/var/log/ceph/cephadm.log: 90.1% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-21T12:45:10.610 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-osd.3.log: 91.5% -- replaced with /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-volume.log.gz 2024-09-21T12:45:10.610 INFO:teuthology.orchestra.run.smithi148.stderr: 91.6% -- replaced with /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-volume.log.gz 2024-09-21T12:45:10.622 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cephadm.log: 91.3% -- replaced with /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-mds.foofs.smithi078.ytpzwh.log.gz 2024-09-21T12:45:10.646 INFO:teuthology.orchestra.run.smithi148.stderr: 89.8% -- replaced with /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-mgr.smithi148.oyiqmx.log.gz 2024-09-21T12:45:10.649 INFO:teuthology.orchestra.run.smithi078.stderr: 91.8% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-21T12:45:10.915 INFO:teuthology.orchestra.run.smithi148.stderr: 92.6% -- replaced with /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-mon.smithi148.log.gz 2024-09-21T12:45:10.968 INFO:teuthology.orchestra.run.smithi078.stderr: 88.7% -- replaced with /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-mgr.smithi078.ntwyan.log.gz 2024-09-21T12:45:12.475 INFO:teuthology.orchestra.run.smithi078.stderr: 91.8% -- replaced with /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-mon.smithi078.log.gz 2024-09-21T12:45:22.173 INFO:teuthology.orchestra.run.smithi078.stderr: 94.5% -- replaced with /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-osd.2.log.gz 2024-09-21T12:45:24.593 INFO:teuthology.orchestra.run.smithi148.stderr: 93.9% -- replaced with /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-osd.4.log.gz 2024-09-21T12:45:25.709 INFO:teuthology.orchestra.run.smithi148.stderr: 94.2% -- replaced with /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-osd.6.log.gz 2024-09-21T12:45:26.581 INFO:teuthology.orchestra.run.smithi078.stderr: 94.1% -- replaced with /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-osd.0.log.gz 2024-09-21T12:45:27.519 INFO:teuthology.orchestra.run.smithi078.stderr: 94.0% -- replaced with /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-osd.3.log.gz 2024-09-21T12:45:28.600 INFO:teuthology.orchestra.run.smithi078.stderr: 94.2% -- replaced with /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-osd.1.log.gz 2024-09-21T12:45:28.602 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-09-21T12:45:28.602 INFO:teuthology.orchestra.run.smithi078.stderr:real 0m18.022s 2024-09-21T12:45:28.602 INFO:teuthology.orchestra.run.smithi078.stderr:user 1m2.246s 2024-09-21T12:45:28.602 INFO:teuthology.orchestra.run.smithi078.stderr:sys 0m2.507s 2024-09-21T12:45:29.611 INFO:teuthology.orchestra.run.smithi148.stderr: 94.2% -- replaced with /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-osd.5.log.gz 2024-09-21T12:45:33.246 INFO:teuthology.orchestra.run.smithi148.stderr: 94.1% -- replaced with /var/log/ceph/6263deca-7813-11ef-baf6-efdc52797490/ceph-osd.7.log.gz 2024-09-21T12:45:33.248 INFO:teuthology.orchestra.run.smithi148.stderr: 2024-09-21T12:45:33.248 INFO:teuthology.orchestra.run.smithi148.stderr:real 0m22.667s 2024-09-21T12:45:33.248 INFO:teuthology.orchestra.run.smithi148.stderr:user 1m8.402s 2024-09-21T12:45:33.248 INFO:teuthology.orchestra.run.smithi148.stderr:sys 0m2.674s 2024-09-21T12:45:33.248 INFO:tasks.cephadm:Archiving logs... 2024-09-21T12:45:33.249 DEBUG:teuthology.misc:Transferring archived files from smithi078:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-09-21_11:23:40-orch:cephadm-wip-guits-main-2024-09-21-0842-distro-default-smithi/7913924/remote/smithi078/log 2024-09-21T12:45:33.250 DEBUG:teuthology.orchestra.run.smithi078:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-21T12:45:41.583 DEBUG:teuthology.misc:Transferring archived files from smithi148:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-09-21_11:23:40-orch:cephadm-wip-guits-main-2024-09-21-0842-distro-default-smithi/7913924/remote/smithi148/log 2024-09-21T12:45:41.584 DEBUG:teuthology.orchestra.run.smithi148:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-21T12:45:50.893 INFO:tasks.cephadm:Removing cluster... 2024-09-21T12:45:50.894 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 6263deca-7813-11ef-baf6-efdc52797490 --force 2024-09-21T12:45:51.252 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 6263deca-7813-11ef-baf6-efdc52797490 --force 2024-09-21T12:45:51.624 INFO:tasks.cephadm:Removing cephadm ... 2024-09-21T12:45:51.624 DEBUG:teuthology.orchestra.run.smithi078:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-21T12:45:51.640 DEBUG:teuthology.orchestra.run.smithi148:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-21T12:45:51.655 INFO:tasks.cephadm:Teardown complete 2024-09-21T12:45:51.655 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-21T12:45:51.671 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-21T12:45:51.671 DEBUG:teuthology.orchestra.run.smithi078:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T12:45:51.683 DEBUG:teuthology.orchestra.run.smithi148:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T12:45:51.695 INFO:teuthology.orchestra.run.smithi078.stderr:bash: line 1: ntpq: command not found 2024-09-21T12:45:51.699 INFO:teuthology.orchestra.run.smithi078.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T12:45:51.699 INFO:teuthology.orchestra.run.smithi078.stdout:=============================================================================== 2024-09-21T12:45:51.699 INFO:teuthology.orchestra.run.smithi078.stdout:^* hv01.front.sepia.ceph.com 3 6 377 52 +100us[ +118us] +/- 40ms 2024-09-21T12:45:51.699 INFO:teuthology.orchestra.run.smithi078.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 50 -432us[ -432us] +/- 69ms 2024-09-21T12:45:51.699 INFO:teuthology.orchestra.run.smithi078.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 52 +356us[ +356us] +/- 70ms 2024-09-21T12:45:51.700 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T12:45:51.709 INFO:teuthology.orchestra.run.smithi148.stderr:bash: line 1: ntpq: command not found 2024-09-21T12:45:51.713 INFO:teuthology.orchestra.run.smithi148.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T12:45:51.713 INFO:teuthology.orchestra.run.smithi148.stdout:=============================================================================== 2024-09-21T12:45:51.713 INFO:teuthology.orchestra.run.smithi148.stdout:^* hv01.front.sepia.ceph.com 3 6 377 50 +93us[ +121us] +/- 40ms 2024-09-21T12:45:51.713 INFO:teuthology.orchestra.run.smithi148.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 50 -444us[ -416us] +/- 69ms 2024-09-21T12:45:51.714 INFO:teuthology.orchestra.run.smithi148.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 51 +351us[ +379us] +/- 70ms 2024-09-21T12:45:51.714 INFO:teuthology.orchestra.run.smithi148.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T12:45:51.715 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-21T12:45:51.724 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-21T12:45:51.725 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-21T12:45:51.736 DEBUG:teuthology.orchestra.run.smithi078:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-21T12:45:51.816 DEBUG:teuthology.orchestra.run.smithi148:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-21T12:45:51.887 DEBUG:teuthology.orchestra.run.smithi078:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T12:45:51.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T12:45:51.915 DEBUG:teuthology.orchestra.run.smithi148:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T12:45:51.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T12:45:51.942 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-21T12:45:51.952 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-21T12:45:51.961 INFO:teuthology.task.internal:Duration was 1965.605973 seconds 2024-09-21T12:45:51.962 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-21T12:45:51.971 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-21T12:45:51.971 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-21T12:45:51.973 DEBUG:teuthology.orchestra.run.smithi148:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-21T12:45:52.009 INFO:teuthology.orchestra.run.smithi078.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T12:45:52.021 INFO:teuthology.orchestra.run.smithi148.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T12:45:52.343 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-21T12:45:52.343 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi078.front.sepia.ceph.com 2024-09-21T12:45:52.343 DEBUG:teuthology.orchestra.run.smithi078:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-21T12:45:52.364 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi148.front.sepia.ceph.com 2024-09-21T12:45:52.364 DEBUG:teuthology.orchestra.run.smithi148:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-21T12:45:52.412 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-21T12:45:52.412 DEBUG:teuthology.orchestra.run.smithi078:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-21T12:45:52.414 DEBUG:teuthology.orchestra.run.smithi148:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-21T12:45:52.478 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-21T12:45:52.478 DEBUG:teuthology.orchestra.run.smithi078:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-21T12:45:52.481 DEBUG:teuthology.orchestra.run.smithi148:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-21T12:46:06.291 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-21T12:46:06.304 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-21T12:46:06.304 DEBUG:teuthology.orchestra.run.smithi078:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-21T12:46:06.329 DEBUG:teuthology.orchestra.run.smithi148:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-21T12:46:06.359 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-21T12:46:06.372 DEBUG:teuthology.orchestra.run.smithi078:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-21T12:46:06.374 DEBUG:teuthology.orchestra.run.smithi148:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-21T12:46:06.395 INFO:teuthology.orchestra.run.smithi078.stdout:kernel.core_pattern = core 2024-09-21T12:46:06.427 INFO:teuthology.orchestra.run.smithi148.stdout:kernel.core_pattern = core 2024-09-21T12:46:06.451 DEBUG:teuthology.orchestra.run.smithi078:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-21T12:46:06.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T12:46:06.499 DEBUG:teuthology.orchestra.run.smithi148:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-21T12:46:06.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T12:46:06.515 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-21T12:46:06.525 INFO:teuthology.task.internal:Transferring archived files... 2024-09-21T12:46:06.526 DEBUG:teuthology.misc:Transferring archived files from smithi078:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-09-21_11:23:40-orch:cephadm-wip-guits-main-2024-09-21-0842-distro-default-smithi/7913924/remote/smithi078 2024-09-21T12:46:06.526 DEBUG:teuthology.orchestra.run.smithi078:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-21T12:46:06.599 DEBUG:teuthology.misc:Transferring archived files from smithi148:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-09-21_11:23:40-orch:cephadm-wip-guits-main-2024-09-21-0842-distro-default-smithi/7913924/remote/smithi148 2024-09-21T12:46:06.600 DEBUG:teuthology.orchestra.run.smithi148:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-21T12:46:06.641 INFO:teuthology.task.internal:Removing archive directory... 2024-09-21T12:46:06.641 DEBUG:teuthology.orchestra.run.smithi078:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-21T12:46:06.643 DEBUG:teuthology.orchestra.run.smithi148:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-21T12:46:06.691 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-21T12:46:06.703 INFO:teuthology.task.internal:Not uploading archives. 2024-09-21T12:46:06.703 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-21T12:46:06.726 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-21T12:46:06.726 DEBUG:teuthology.orchestra.run.smithi078:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-21T12:46:06.728 DEBUG:teuthology.orchestra.run.smithi148:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-21T12:46:06.741 INFO:teuthology.orchestra.run.smithi078.stdout: 265324 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 21 12:46 /home/ubuntu/cephtest 2024-09-21T12:46:06.746 INFO:teuthology.orchestra.run.smithi148.stdout: 265324 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 21 12:46 /home/ubuntu/cephtest 2024-09-21T12:46:06.748 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-21T12:46:06.760 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-21T12:46:06.783 INFO:teuthology.run:Summary data: description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 1965.6059730052948 owner: scheduled_gabrioux@teuthology success: true 2024-09-21T12:46:06.783 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-21T12:46:06.957 INFO:teuthology.run:pass